def build_graph():
G = nx.Graph()
G.add_nodes_from(shared.bidders.keys())
user_to_auc = defaultdict(set)
# for auc, bids in shared.auction_to_bids.iteritems():
# users = list(set([bid.bidder_id for bid in bids]))
# n = len(users)
# for i in xrange(n):
# for j in xrange(n):
# G.add_edge(users[i], users[j])
for auc, bids in shared.auction_to_bids.iteritems():
for bid in bids:
user_to_auc[bid.bidder_id].add(auc)
pair_to_user = defaultdict(set)
for bidder_id, aucs in user_to_auc.iteritems():
n = len(aucs)
a = list(aucs)
for i in xrange(n):
for j in xrange(i + 1, n):
u = min(a[i], a[j])
v = max(a[i], a[j])
pair_to_user[(u, v)].add(bidder_id)
for pair, bidders in pair_to_user:
n = len(bidders)
b = list(bidders)
for i in xrange(n):
for j in xrange(i + 1, n):
G.add_edge(bidders[i], bidders[j])
utils.draw_graph(G, shared.bidders)
def train():
#torchのnnを使ってグラフを構築
model = line.Model_torch()
if torch.cuda.is_available():
tt = torch.cuda
model.cuda()
else:
tt = torch
#torchのnnを使ってグラフを構築
model = line.Model_torch()
#torchのnnを使わずに、グラフを自定義
#model=line.Model_my(tt)
#最適化関数
optimizer = model.optimizer
x, t, x_data, y_data = set_data(tt)
# Training: forward, loss, backward, step
# Training loop
start = time.time()
plt.ion()
w, b = 0, 0
for step in range(hparams.train_steps + 1):
# Forward pass
y_pred = model.linear_model(x_data)
loss = model.loss(y_pred, y_data)
if step % hparams.valid_steps == 0 or step == hparams.valid_steps:
w, b = out_log(step, loss, model)
# Visualization of learning process
y = utils.liner(x, w, b)
plt.plot(x, t, 'o', label="dots")
plt.plot(x, y, label="line")
plt.xlabel("x")
plt.ylabel("y")
plt.axis([-0.5, 2, -0.5, 2])
plt.title('linear regression for PyToch: train of step :%s' % step)
plt.pause(0.5)
plt.cla()
# Zero gradients
optimizer.zero_grad()
#勾配を計算、逆伝播
loss.backward()
# update weights
optimizer.step()
plt.ioff()
save(model)
print('train time: %.5f' % (time.time() - start))
print('weight: %s bias: %s loss: %s' % (w, b, loss))
utils.draw_graph("Pytoch", x, utils.liner(x, w, b), w, b, t, step)
def main():
from utils import draw_graph
xs = list(range(20))
ys = list(range(20))
areas = [
Area([(x, y), (x + 1, y + 1)], Node.FREE) for (x, y) in zip(xs, ys)
]
g = Grid()
g.from_poly(areas)
graph = g.to_graph()
draw_graph(graph)
def test(search_f, edges=10, nodes=10, draw=True):
print('Creating graph')
graph = create_graph(edges, nodes)
print('Testing', search_f.__name__)
start = time.time()
result = search_f(graph)
end = time.time()
print('{} with {} nodes and {} edges took {:.3f}s'.format(
search_f.__name__, nodes, edges, end - start))
print('Final result', result)
if draw:
draw_graph(graph, result[0])
def search_engine_3(query_match):
print('In which year was the movie released?')
year_user = int(input())
# Rank the results by closeness to a given year
years = utils.year_docs(query_match)
sim_years = utils.sim_docs(years, year_user)
df = pd.DataFrame(columns=['Title','Intro','Wikipedia Url', 'Similarity'])
for sim in heapq.nlargest(5, sim_years.items(), key = lambda i: i[1]):
i = sim[0] # document_id
file = open('webpages/tsv/output_%d.tsv' %i).read().split('\n\n')[1].split('\t')
title, intro, link = file[3].encode('utf8').decode("unicode_escape"), file[1].encode('utf8').decode("unicode_escape"), urls[str(i+1)]
new_row = {'Title':title, 'Intro': intro, 'Wikipedia Url': link, 'Similarity': sim[1]}
df = df.append(new_row, ignore_index=True)
# Visualization of the top 5 documents related to the query
d = dict(selector="th", props=[('text-align', 'center')])
df1 = df.sort_values(by=['Similarity'], ascending = False)
df1.style.format({'Wikipedia Url': utils.make_clickable}).hide_index().set_table_styles([d]).set_properties(**{'text-align': 'center'}).set_properties(subset=['Title'], **{'width': '130px'})
# Bonus: CO-STARDOM NETWORK
movies = [movie[0] for movie in heapq.nlargest(10, sim_years.items(), key = lambda i: i[1])]
G = utils.add_nodes(movies)
G = utils.add_edges(G)
network = utils.draw_graph(G)
return df, network
def run(graph, start, goal, search_alg=None):
global screen, edges, clock, font, global_graph
global_graph = graph
# add start colors to graph
for element in global_graph:
element.extend([grey, black])
# print(global_graph)
build_edges()
pygame.init()
clock = pygame.time.Clock()
screen = pygame.display.set_mode((display_width, display_height))
font = pygame.font.Font(pygame.font.get_default_font(), 25)
pygame.display.set_caption('Search on Graph - ' + str(search_alg).upper())
draw_graph(screen, font, global_graph, edges) # initial
updateUI()
pygame.time.delay(delay_start_search_time) # wait 5 sec to start
if search_alg == 'bfs':
BFS(global_graph, edges, edge_id, start, goal)
elif search_alg == 'dfs':
DFS(global_graph, edges, edge_id, start, goal)
elif search_alg == 'ucs':
UCS(global_graph, edges, edge_id, start, goal)
elif search_alg == 'a_star':
AStar(global_graph, edges, edge_id, start, goal)
elif search_alg == 'greedy':
GreedySearch(global_graph, edges, edge_id, start, goal)
elif search_alg == 'bidirection':
BidirectionalSearch(graph, edges, edge_id, start, goal)
elif search_alg == 'ids':
IDS(graph, edges, edge_id, start, goal)
elif search_alg == 'beamsearch':
BeamSearch(graph, edges, edge_id, start, goal)
# elif search_alg == 'other_algorithm':
# pass
else:
print("Pass a search algorithm to run program.")
example_func(global_graph, edges, edge_id, start, goal)
while True:
quit_event()
def train():
x, t = utils.loadData()
# Set the initial weight parameter
w = 0.1 #2
b = -0.1 #-.5
wb_cost = [(w, b, cost(nn(x, w, b),
t))] # List to store the weight,costs values
start = time.time()
plt.ion()
for step in range(hparams.train_steps + 1):
dw, db = delta_wb(w, x, b, t,
hparams.learning_rate) # Get the delta w update
w = w - dw # Update the current weight parameter
b = b - db
wb_cost.append((w, b, cost(nn(x, w, b), t))) # Add weight,cost to list
loss = str(wb_cost[step][2])
if step % hparams.valid_steps == 0 or step == hparams.train_steps:
print("step:%s" % step, 'loss:%s' % loss)
weight = wb_cost[step][0]
bias = wb_cost[step][1]
dict = {"weight": weight, "bias": bias}
#save model
with open(model_path, "w") as f:
json.dump(dict, f)
#Visualization of learning process
y = nn(x, w, b)
plt.plot(x, t, 'o', label="dots")
plt.plot(x, y, label="line")
plt.xlabel("x")
plt.ylabel("y")
plt.axis([-0.5, 2, -0.5, 2])
plt.title('linear regression for Numpy: train of step :%s' % step)
plt.pause(0.5)
plt.cla()
plt.ioff()
print('train time: %.5f' % (time.time() - start))
print('weight: %s bias: %s loss: %s' % (weight, bias, loss))
utils.draw_graph("Numpy", x, nn(x, w, b), w, b, t, step)
def run(regular_expression):
regular_expression = preprocess_regular_expression(regular_expression)
is_valid, operations = validate_and_parse(regular_expression)
print(operations)
if is_valid:
# Convert to corresponding NFA
json = construct_nfa(operations)
print(json)
try:
draw_graph(json)
except:
print("Error in drawing graph")
file = open("output.json", "w", encoding="utf-8")
file.write(json)
file.close()
else:
# The regular expression is invalid
print("The regular expression is invalid.")
def main():
random.seed(318)
pop = toolbox.population(n=300)
hof = tools.HallOfFame(1)
stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
stats_size = tools.Statistics(len)
mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
mstats.register("min", np.min)
mstats.register("avg", np.mean)
pop, log = algorithms.eaSimple(pop,
toolbox,
0.5,
0.2,
20,
stats=mstats,
halloffame=hof,
verbose=True)
expr = hof[0]
tree = gp.PrimitiveTree(expr)
print('LOSS: {}'.format(evalByColorProportion(expr)))
print()
print('CODE: {}'.format(tree))
print()
func = toolbox.compile(expr=expr)
markup = func(x='text')
print('HTML: \n')
print(markup)
print()
draw_graph(expr)
draw_logbook(log)
func = toolbox.compile(expr=expr)
markup = body(func(x='text'))
result_img = renderer.render_html(markup)
result_img.save('output/result_html.png')
return pop, log, hof
def test(search_fs, edges=10, nodes=10, draw=True):
print('Creating graph')
graph = create_graph(edges, nodes)
for search_f in search_fs:
print()
print('Testing', search_f.__name__)
start = time.time()
result = search_f(graph)
end = time.time()
print('{3} with {0} nodes and {1} edges took {2:.3f}s'.format(
nodes, edges, end - start, search_f.__name__))
print('Final result', result)
if draw:
draw_graph(graph)
for (v, w) in graph.edges:
graph[v][w]['in_matching'] = False
def main():
from data import Area, Node, Grid
from utils import draw_graph, draw_path
xs = list(range(90))
ys = list(range(90))
areas = [
Area([(x, y), (x + 1, y + 1)], Node.VICTIM) for (x, y) in zip(xs, ys)
]
g = Grid()
g.from_poly(areas)
graph = g.to_graph()
target = graph.rows[-1][-1]
target.kind = target.TARGET
clean(graph)
init_dist(graph, target)
from time import time
print("started")
t = time()
path = find_path(graph.rows[0][0], target, cf)
print((time() - t), "s")
draw_path(path)
draw_graph(graph)
def updateUI():
global screen, edges, clock, font
draw_graph(screen, font, global_graph, edges)
pygame.display.update()
clock.tick(fps_speed)
break
# Unpack batch
mks, nds, eds, nd_to_sample, ed_to_sample = batch
# Configure input
real_mks = Variable(mks.type(Tensor))
given_nds = Variable(nds.type(Tensor))
given_eds = eds
# Sample noise as generator input
layouts_imgs_tensor = []
# draw graph
graph_img = draw_graph(nds.detach().cpu().numpy(),
eds.detach().cpu().numpy(),
0,
im_size=256)
all_imgs.append(graph_img)
# reconstruct
for j in range(opt.num_variations):
z_shape = [real_mks.shape[0], opt.latent_dim]
z = Variable(Tensor(np.random.normal(0, 1, tuple(z_shape))))
with torch.no_grad():
gen_mks = generator(z, given_nds, given_eds)
gen_bbs = np.array(
[np.array(mask_to_bb(mk)) for mk in gen_mks.detach().cpu()])
real_bbs = np.array(
[np.array(mask_to_bb(mk)) for mk in real_mks.detach().cpu()])
real_nodes = np.where(given_nds.detach().cpu() == 1)[-1]
column_restriction = 0
for j in range(row_count):
column_restriction += edges[i][j]
mdl.add(column_restriction == 1)
# Row restriction
for i in range(row_count):
row_restriction = 0
for j in range(row_count):
row_restriction += edges[j][i]
mdl.add(row_restriction == 1)
# Objective function
obj = 0
for i in range(row_count):
for j in range(row_count):
obj += edges[i][j] * distances[i][j]
mdl.minimize(obj)
# Add callback
cb = mdl.register_callback(Callback)
# Add attributes to Callback object, so i can access them inside de Callback
cb.row_count = row_count
slv = mdl.solve()
# Draw the graph
utils.draw_graph(data, edges)
# a) Construa uma arvore de regressao usando a medida de reducao de desvio
# padrao e um criterio de pre-poda (pode ser um valor minimo de reducao de
# desvio padrao). Selecione aleatoriamente 75% dos dados para treinamento.
# Retorne a estrutura da arvore construida.
nclasses = np.union1d(y, y).size
n = len(y)
randind = np.arange(0, n)
np.random.shuffle(randind)
ind_train = randind[0:0.75 * n]
ind_test = randind[0.75 * n:n]
tree = RegressionTree(nclasses)
tree.train(x[ind_train, :], y[ind_train], SDRMIN=0.1, NMIN=3)
g, pos = tree.gerar_grafo()
utils.draw_graph(g, pos)
# b) Use os restantes 25% dos dados para avaliacao. Retorne as medidas MAPE e
# RMSE.
yhat = tree.estimate(x[ind_test, :])
rmse = utils.rmse(y[ind_test], yhat)
mape = utils.mape(y[ind_test], yhat)
print 'RMSE encontrado: {:3.2f}\nMAPE encontrado: {:3.2f}'.format(rmse, mape)
plt.plot(y[ind_test])
plt.hold(True)
plt.plot(yhat)
plt.legend(['real', 'estimado'])
plt.show()
def draw(self, graph_name="tmp", file_name="tmp"):
connects = []
hints = {}
self.__dfs(self.root, connects, hints)
draw_graph(connects, hints, graph_name=graph_name, file_name=file_name)
def set_delta_tau_k_ij(k, K, Je):
for j, i in K:
if Metric[j][i] and Je:
tau_all[k][j][i] = 1 / Je
else:
tau_all[k][j][i] = 0
def set_delta_tau_best_ij(K, Je):
global Je_best
if Je_best == 0 or Je_best > Je:
Je_best = Je
tau_best[...] = .0
for j, i in K:
tau_best[j][i] = Je
if __name__ == "__main__":
t = time.process_time()
run_aco_algorithm()
elapsed_time = time.process_time() - t
print("Time")
print(elapsed_time)
print("Best path value")
print(Je_best_t[-1][1])
print("Liczba iteracji")
print(Je_best_t[-1][0])
utils.save(get_tau(), get_path(), Je_best_t, Metric, elapsed_time,
'out_seq')
utils.draw_graph(utils.get_adjency_matrix(Metric), True, 'graph5x5.png')
def DFS_limitedDepth(graph, edges, edge_id, start, goal, limited):
"""
DFS search
"""
# TODO:
clock = pygame.time.Clock()
screen = pygame.display.set_mode((display_width, display_height))
font = pygame.font.Font(pygame.font.get_default_font(), 25)
print("Implement Depth-Limited-Search algorithm: limited =", limited)
n = len(graph)
weightedMatrix = np.ones((n, n))
parent = {}
parent[start] = -1
if start == goal:
graph[goal][3] = purple
graph[goal][2] = white
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
return True
#-----#-----#-----#-----#-----#-----
frontier = []
frontier.append([start, 0])
explored = set()
while True:
if len(frontier) == 0:
print("Can't find the goal!!!")
return False
node = frontier.pop()
depth = node[1]
explored.add(node[0])
if depth >= limited:
continue
graph[node[0]][3] = yellow
graph[node[0]][2] = white
#Draw copy graph
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
for successor in graph[node[0]][1]:
if (successor not in explored) and (successor not in [
i[0] for i in frontier
]):
parent[successor] = node[0]
edges[edge_id(node[0], successor)][1] = white
graph[successor][2] = white
graph[successor][3] = red
#Draw copy graph
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
if successor == goal:
child = goal
graph[child][2] = white
graph[child][3] = purple
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
PathCost = 0
while parent[child] != -1:
PathCost = PathCost + weightedMatrix[
parent[child]][child]
edges[edge_id(parent[child], child)][1] = green
child = parent[child]
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
if child != goal:
graph[child][3] = orange
draw_graph(screen, font, graph, edges)
pygame.display.update()
clock.tick(fps_speed)
return True
frontier.append([successor, depth + 1])
graph[node[0]][3] = blue
pass
for i in range(settings.FRONT_SIZE + settings.BACK_SIZE):
probabilities.append(predicts[i][j])
# 根据概率分布随机选择一个序列
balls = utils.select_seqs(probabilities)
# 计算奖金
award = utils.lotto_calculate(outputs, balls)
money_in += award
if award:
print('{} 中奖了,{}元！ {}/{}'.format(j, award, money_in, money_out))
print('买彩票花费金钱共{}元，中奖金额共{}元，赚取{}元'.format(money_out, money_in, money_in - money_out))
return money_in - money_out
# 初始化数据集
lotto_dataset = LottoDataSet(train_data_rate=0.9)
# 创建保存权重的文件夹
if not os.path.exists(settings.CHECKPOINTS_PATH):
os.mkdir(settings.CHECKPOINTS_PATH)
# 开始训练
results = []
for epoch in range(1, settings.EPOCHS + 1):
model.fit(lotto_dataset.train_np_x, lotto_dataset.train_np_y, batch_size=settings.BATCH_SIZE, epochs=1)
# 保存当前权重
model.save_weights('{}/model_checkpoint_{}'.format(settings.CHECKPOINTS_PATH, epoch))
print('已训练完第{}轮，尝试模拟购买彩票...'.format(epoch))
results.append(simulate(lotto_dataset.test_np_x, lotto_dataset.test_np_y))
# 输出每一轮的模拟结果
print(results)
# 显示每一轮模拟结果的变化趋势
utils.draw_graph(results)
def train(args, dataset_train, rnn, output):
# check if load existing model
if args.load:
fname = (args.model_save_path + args.fname + "lstm_" +
str(args.load_epoch) + ".dat")
rnn.load_state_dict(torch.load(fname))
fname = (args.model_save_path + args.fname + "output_" +
str(args.load_epoch) + ".dat")
output.load_state_dict(torch.load(fname))
args.lr = 0.00001
epoch = args.load_epoch
print("model loaded!, lr: {}".format(args.lr))
else:
epoch = 1
# initialize optimizer
optimizer_rnn = optim.Adam(list(rnn.parameters()), lr=args.lr)
optimizer_output = optim.Adam(list(output.parameters()), lr=args.lr)
scheduler_rnn = MultiStepLR(optimizer_rnn,
milestones=args.milestones,
gamma=args.lr_rate)
scheduler_output = MultiStepLR(optimizer_output,
milestones=args.milestones,
gamma=args.lr_rate)
# start main loop
time_all = np.zeros(args.epochs)
while epoch <= args.epochs:
time_start = tm.time()
# train
if "GraphRNN_VAE" in args.note:
train_vae_epoch(
epoch,
args,
rnn,
output,
dataset_train,
optimizer_rnn,
optimizer_output,
scheduler_rnn,
scheduler_output,
)
elif "GraphRNN_MLP" in args.note:
train_mlp_epoch(
epoch,
args,
rnn,
output,
dataset_train,
optimizer_rnn,
optimizer_output,
scheduler_rnn,
scheduler_output,
)
elif "GraphRNN_RNN" in args.note:
train_rnn_epoch(
epoch,
args,
rnn,
output,
dataset_train,
optimizer_rnn,
optimizer_output,
scheduler_rnn,
scheduler_output,
)
time_end = tm.time()
time_all[epoch - 1] = time_end - time_start
# test
if epoch % args.epochs_test == 0 and epoch >= args.epochs_test_start:
for sample_time in range(1, 4):
G_pred = []
while len(G_pred) < args.test_total_size:
if "GraphRNN_VAE" in args.note:
G_pred_step = test_vae_epoch(
epoch,
args,
rnn,
output,
test_batch_size=args.test_batch_size,
sample_time=sample_time,
)
elif "GraphRNN_MLP" in args.note:
G_pred_step = test_mlp_epoch(
epoch,
args,
rnn,
output,
test_batch_size=args.test_batch_size,
sample_time=sample_time,
)
elif "GraphRNN_RNN" in args.note:
G_pred_step = test_rnn_epoch(
epoch,
args,
rnn,
output,
test_batch_size=args.test_batch_size,
)
G_pred.extend(G_pred_step)
# save graphs
fname = (args.graph_save_path + args.fname_pred + str(epoch) +
"_" + str(sample_time) + ".dat")
save_graph_list(G_pred, fname)
draw_graph(random.choice(G_pred), prefix=f"collagen-{epoch}")
if "GraphRNN_RNN" in args.note:
break
print("test done, graphs saved")
# save model checkpoint
if args.save:
if epoch % args.epochs_save == 0:
fname = (args.model_save_path + args.fname + "lstm_" +
str(epoch) + ".dat")
torch.save(rnn.state_dict(), fname)
fname = (args.model_save_path + args.fname + "output_" +
str(epoch) + ".dat")
torch.save(output.state_dict(), fname)
epoch += 1
np.save(args.timing_save_path + args.fname, time_all)
import networkx as nx
import matplotlib.pyplot as plt
import paths_dag
import opt_sni
import parse
import graph_tools
from utils import draw_graph
# like formatted but removes NOT operations
s = open('fantomas/f2.txt').read()
g = parse.parse(s, tag_output_fn=lambda g, n: n.startswith('y'))
draw_graph(g)
plt.show()
split_c_d = graph_tools.add_split_nodes(g)
split_c = list(split_c_d.values())
print('baseline cut', sum(len(x) * (len(x) - 1) for x in split_c))
draw_graph(g)
plt.show()
cut_edges = opt_sni.opt_sni(g, split_c, max_seconds=60)
g2 = graph_tools.without_edges(g, cut_edges)
print('Cut is NI', paths_dag.is_graph_NI(g2))
print('Cut is SNI', paths_dag.is_graph_SNI(g2))
import opt_sni
import parse
import graph_tools
from utils import draw_graph
s = open('repr_aes_bitslice/non_lin.txt').read()
s_out = open('repr_aes_bitslice/lin_out.txt').read()
g_out = parse.parse_string_graph(s_out)
out_nodes = set(g_out.nodes)
g = parse.parse(s, tag_output_fn=lambda g, n: n in out_nodes)
split_c_d = graph_tools.add_split_nodes(g)
split_c = list(split_c_d.values())
print('baseline cut', sum(len(x)*(len(x)-1) for x in split_c))
cut_edges = opt_sni.opt_sni(g, split_c, max_seconds=60*15)
g2 = graph_tools.without_edges(g, cut_edges)
print('Cut is NI:', paths_dag.is_graph_NI(g2))
print('Cut is SNI:', paths_dag.is_graph_SNI(g2))
#draw_graph(g, cut_edges)
#plt.show()
g4, simplified_cut_edges = graph_tools.without_unncessary_splits(g, cut_edges, split_c_d)
draw_graph(g4, simplified_cut_edges)
plt.show()
# a) Construa uma arvore de regressao usando a medida de reducao de desvio
# padrao e um criterio de pre-poda (pode ser um valor minimo de reducao de
# desvio padrao). Selecione aleatoriamente 75% dos dados para treinamento.
# Retorne a estrutura da arvore construida.
nclasses = np.union1d(y, y).size
n = len(y)
randind = np.arange(0, n)
np.random.shuffle(randind)
ind_train = randind[0:0.75 * n]
ind_test = randind[0.75 * n:n]
tree = RegressionTree(nclasses)
tree.train(x[ind_train, :], y[ind_train], SDRMIN=0.1, NMIN=3)
g, pos = tree.gerar_grafo()
utils.draw_graph(g, pos)
# b) Use os restantes 25% dos dados para avaliacao. Retorne as medidas MAPE e
# RMSE.
yhat = tree.estimate(x[ind_test, :])
rmse = utils.rmse(y[ind_test], yhat)
mape = utils.mape(y[ind_test], yhat)
print 'RMSE encontrado: {:3.2f}\nMAPE encontrado: {:3.2f}'.format(rmse,mape)
plt.plot(y[ind_test])
plt.hold(True)
plt.plot(yhat)
plt.legend(['real','estimado'])
plt.show()
def test():
g = nx.MultiDiGraph()
g.add_nodes_from(nodes)
g.add_edges_from(edges)
utils.draw_graph(g)
plt.show()
