def upload(name, networks, onestop_ids, directoryPath): G, node_dict = create_node_network(networks, directoryPath) # Verify that nodes were created properly. # for NI in G.Nodes(): # nodeID = NI.GetId() # print 'Node ID %d, stop_id %s, stop_name %s, stop_code %s, stop_lat %f, stop_lon %f, network_name %s' % (nodeID, G.GetStrAttrDatN(NI, 'stop_id'), G.GetStrAttrDatN(NI, 'stop_name'), G.GetStrAttrDatN(NI, 'stop_code'), G.GetFltAttrDatN(NI, 'stop_lat'), G.GetFltAttrDatN(NI, 'stop_lon'), G.GetStrAttrDatN(NI, 'network_name')) # print 'Node ID %d, stop_name %s, routes %s' % (nodeID, G.GetStrAttrDatN(NI, 'stop_name'), G.GetStrAttrDatN(NI, 'routes')) G = create_l_space_graph(G, node_dict, networks, directoryPath) G, internetwork_edges = get_internetwork_edges(G, 100) snapFileName = "snapData/uploads/" + name.replace(" ", "") + '.graph' pickleFileName = "snapData/uploads/" + name.replace(" ", "") + '.pkl' print snapFileName print pickleFileName try: print('Saving graph...') save_graph(G, snapFileName) with open(pickleFileName, 'wb+') as output: pickle.dump(node_dict, output, pickle.HIGHEST_PROTOCOL) pickle.dump(internetwork_edges, output, pickle.HIGHEST_PROTOCOL) print('Done!') snapPickle = SnapPickle(name=name, networks=networks, onestop_ids=onestop_ids, pub_date=timezone.now(), snapFileName=snapFileName, pickleFileName=pickleFileName) snapPickle.save() return snapPickle except IOError: print('Could not save node network!')
def proceed(self): config = self.pipeline.version.config graph = utils.load_graph('labeled-import.pickle', config) artists = graph.get_nodes()['Artist'].values() # Artists without a category graph.prune([a for a in artists if len(a.categories) < 1]) utils.save_graph(graph, 'labeled-import-prune', config) self.pipeline.update()
def proceed(self): print(f"Proceeding to execute import foo") moma = Moma(constants.MOMA_EXHIBITIONS_CSV) dome = Dome(constants.DOME_ARTISTS_CSV, constants.DOME_EXHIBITIONS_CSV) importer = Importer(moma, dome) importer.run() utils.save_graph(importer.graph, 'import.pickle', self.pipeline.version.config) self.pipeline.update()
def create_data_for_Graphsage(G, num_nodes_label): num_nodes = len(G.nodes()) features = np.zeros((num_nodes, num_nodes_label)) for node in G.nodes: features[node][G.nodes[node]['label']] = 1 graphsage_G = nx.Graph() graphsage_G.add_nodes_from([str(node) for node in G.nodes]) graphsage_G.add_edges_from([(str(edge[0]), str(edge[1])) for edge in list(G.edges)]) features = np.ones((num_nodes, args.feat_dim), dtype=float) id2idx = {node: int(node) for node in list(graphsage_G.nodes)} save_graph(features, graphsage_G, id2idx, args.data_name, args.dir) graph_data = load_data(args.prefix) return graph_data
def proceed(self): config = self.pipeline.version.config graph = utils.load_graph('labeled-import.pickle', config) nodes_by_type = graph.get_nodes() exhibitions = nodes_by_type['Exhibition'].values() artists = nodes_by_type['Artist'].values() # Artists with fewer than 3 exhibitions graph.prune([a for a in artists if a.degrees < 3]) # Exhibitions with fewer than 2 artists graph.prune([e for e in exhibitions if e.degrees < 2]) utils.save_graph(graph, 'labeled-import-prune', config) self.pipeline.update()
def run(g, save=None, show_wd=False): cpg = cp(g) print(f'The original graph has a clock period of {cpg}') print('Running algorithm OPT1') g1 = opt1(g, show_wd=show_wd) if save is not None: path = save+'_opt1.dot' save_graph(g1, path) print(f'Output graph saved to {path}') cpr1 = cp(g1) print(f'The graph returned by OPT1 has a clock period of {cpr1}') print('Running algorithm OPT2') g2 = opt2(g, show_wd=show_wd) if save is not None: path = save+'_opt2.dot' save_graph(g2, path) print(f'Output graph saved to {path}') cpr2 = cp(g2) print(f'The graph returned by OPT2 has a clock period of {cpr2}')
def gen_random_circuit(V=8, E=11, save=False): """ Generate a random synchronous circuit. :param V: The number of nodes. :param E: The number of edges. :param save: If different from ``None`` or ``False``, the path where to save the generated graph. :return: The generated graph. """ while True: g = nx.gnm_random_graph(V, E, directed=True) for v in g.nodes: g.nodes[v]['weight'] = np.random.randint(1, 10) g.nodes[0]['weight'] = 0 for e in g.edges: g.edges[e]['weight'] = np.random.randint(10) if check_if_synchronous_circuit(g): if save: save_graph(g, save) return g
def test_timing_formulation(self, num): runs = 1000 X = [0, 1] Y = [0, 1, 2] # for i in range(2): # X.append(random.randInt(0,2)) # Y = deep_copy(X) # for in range(2): # X.append(random.randInt(4,)) v = [6] inf_R_x = 0 inf_R_y = 0 inf_x = 0 inf_y = 0 for i in range(runs): T_1, T_a, T_b = runIC_fair_timings( (self.G, X + v, self.gamma_a, self.gamma_a)) T_2, T_a, T_b = runIC_fair_timings( (self.G, X, self.gamma_a, self.gamma_a)) inf_x += T_2 inf_R_x += (T_1 - T_2) T_1, T_a, T_b = runIC_fair_timings( (self.G, Y + v, self.gamma_a, self.gamma_a)) T_2, T_a, T_b = runIC_fair_timings( (self.G, Y, self.gamma_a, self.gamma_a)) inf_y += T_2 inf_R_y += (T_1 - T_2) print( ' X : {inf_x / runs}, diff: {inf_R_x / runs} Y: {inf_y / runs} diff: {inf_R_y / runs}' ) if inf_R_x / runs < inf_R_y / runs: ut.save_graph(self.filename + '_violates_{num}_.txt', self.G) nx.draw(self.G) plt.savefig("graph" + '_violates_{num}_.png') print(" \n \n ********** VIOLATED *********** \n\n\n")
def gen_provided_correlator(n, save=False): """ Generate a correlator circuit like the ones described in the paper. :param n: The values 1 or 2, depending on which correlator you want to generate. :param save: Whether to save the generated graph or not. :return: The generated graph. """ g = nx.MultiDiGraph() add_weighted_node(g, 'h', 0) add_weighted_node(g, 'd0', 3) add_weighted_node(g, 'd1', 3) add_weighted_node(g, 'd2', 3) add_weighted_node(g, 'd3', 3) add_weighted_node(g, 'p0', 7) add_weighted_node(g, 'p1', 7) add_weighted_node(g, 'p2', 7) if n == 1: g.add_weighted_edges_from([('h', 'd0', 1), ('d0', 'd1', 1), ('d0', 'p0', 0), ('d1', 'd2', 1), ('d1', 'p1', 0), ('d2', 'd3', 1), ('d2', 'p2', 0), ('d3', 'p2', 0), ('p2', 'p1', 0), ('p1', 'p0', 0), ('p0', 'h', 0)]) if save: save_graph(g, '../graphs/correlator1.dot') elif n == 2: g.add_weighted_edges_from([('h', 'd0', 1), ('d0', 'd1', 1), ('d0', 'p0', 0), ('d1', 'd2', 0), ('d1', 'p1', 0), ('d2', 'd3', 1), ('d2', 'p2', 0), ('d3', 'p2', 0), ('p2', 'p1', 1), ('p1', 'p0', 0), ('p0', 'h', 0)]) if save: save_graph(g, '../graphs/correlator2.dot') else: raise NotImplementedError() return g
def proceed(self): config = self.pipeline.version.config report_path = os.path.join(config['version_dir'], 'fetch-report.json') graph = utils.load_graph('import.pickle', config) nodes = [value for value in graph if value.type == 'Artist' and value.wikidataID != None and value.degrees > 2] # For rerunning the process without restarting, use # the nodes from the report, using `report_path` above. # nodes = [graph[tk] for tk in report['unupdated']] fetch = FetchLabels(nodes, graph, config) (new_graph, report) = fetch.run() utils.save_graph(new_graph, 'labeled-import.pickle', config) with open(report_path, 'wb') as f: f.write(json.dumps(report)) self.pipeline.update()
def gen_correlator(k, save=False): """ Generate a correlator of order :math:`k`. :param k: The order of the correlator. :param save: Whether to save the generated graph or not. :return: The generated graph. """ assert k >= 1, 'k should be greater than or equal to 1' g = nx.DiGraph() add_weighted_node(g, 'h', 0) for i in range(k + 1): add_weighted_node(g, f'd{i}', 3) if i < k: add_weighted_node(g, f'p{i}', 7) g.add_weighted_edges_from([('h', 'd0', 1)]) g.add_weighted_edges_from([(f'd{i}', f'd{i+1}', 1) for i in range(k)]) g.add_weighted_edges_from([(f'd{i}', f'p{i}', 0) for i in range(k)]) g.add_weighted_edges_from([(f'd{k}', f'p{k-1}', 0)]) g.add_weighted_edges_from([(f'p{i+1}', f'p{i}', 0) for i in range(k - 1)]) g.add_weighted_edges_from([('p0', 'h', 0)]) if save: save_graph(g, f'../graphs/correlator_k{k}.dot') return g
def create_and_save_graph(xnodes, ynodes, x_weights_idmat_nodeff, y_weights_idmat_nodeff, color, dir_path, filename, model_dat, colortrans=None): """create graph as dot string, save it as png and return it as svg""" form = (" node [style=rounded]\n" " node [shape=box]\n" ' ratio="compress"\n') if color is True: base = compute_color_base([ # absolute max over all values x_weights_idmat_nodeff[0], x_weights_idmat_nodeff[2], y_weights_idmat_nodeff[0], y_weights_idmat_nodeff[2] ]) elif color is None: base = None else: base = abs(color) # e.g. color = 2 for t-values x_dot = dot(xnodes, ynodes, *x_weights_idmat_nodeff, color, base, colortrans, filename, model_dat) y_dot = dot(ynodes, ynodes, *y_weights_idmat_nodeff, color, base, colortrans, filename, model_dat) dot_str = "digraph { \n" + form + x_dot + y_dot + " }" utils.save_graph(dir_path, filename, dot_str) graph_svg = utils.render_dot(dot_str, out_type="svg") return graph_svg
def __init__(self, args, num_classes=1000): super(CNN, self).__init__() self.conv1 = depthwise_separable_conv_3x3(3, args.channels // 2, 2) self.bn1 = nn.BatchNorm2d(args.channels // 2) if args.net_type == 'small': self.conv2 = Triplet_unit(args.channels // 2, args.channels, 2) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv3.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv3.yaml')) self.conv3 = StageBlock(graph, args.channels, args.channels) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv4.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv4.yaml')) self.conv4 = StageBlock(graph, args.channels, args.channels * 2) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv5.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv5.yaml')) self.conv5 = StageBlock(graph, args.channels * 2, args.channels * 4) self.relu = nn.ReLU() self.conv = nn.Conv2d(args.channels * 4, 1280, kernel_size=1) self.bn2 = nn.BatchNorm2d(1280) elif args.net_type == 'regular': if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv2.yaml')) else: graph = build_graph(args.nodes // 2, args) save_graph(graph, os.path.join(args.model_dir, 'conv2.yaml')) self.conv2 = StageBlock(graph, args.channels // 2, args.channels) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv3.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv3.yaml')) self.conv3 = StageBlock(graph, args.channels, args.channels * 2) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv4.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv4.yaml')) self.conv4 = StageBlock(graph, args.channels * 2, args.channels * 4) if args.resume: graph = load_graph(os.path.join(args.model_dir, 'conv5.yaml')) else: graph = build_graph(args.nodes, args) save_graph(graph, os.path.join(args.model_dir, 'conv5.yaml')) self.conv5 = StageBlock(graph, args.channels * 4, args.channels * 8) self.relu = nn.ReLU() self.conv = nn.Conv2d(args.channels * 8, 1280, kernel_size=1) self.bn2 = nn.BatchNorm2d(1280) self.avgpool = nn.AvgPool2d(7, stride=1) self.fc = nn.Linear(1280, num_classes) for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n))
from networks import Exp1, Exp2, Exp3, Exp4, Exp5, Exp6 from utils import save_graph net1 = Exp1(n_layers=3) save_graph('results/exp1.dot', (8, 16), net1) net2 = Exp2(n_layers=3) save_graph('results/exp2.dot', (8, 16), net2) net3 = Exp3(n_layers=3) save_graph('results/exp3.dot', (8, 16), net3) net4 = Exp4(n_layers=5) save_graph('results/exp4.dot', (8, 16), net4) net5 = Exp5(n_layers=3) save_graph('results/exp5.dot', (8, 3, 64, 64), net5) net6 = Exp6(n_layers=3) save_graph('results/exp6.dot', (8, 3, 64, 64), net6)
def generate_reports(organization_name, project, base_dir, weeks, issues_per_week, comments_per_week, comment_avg_len_per_week, comments_len_per_week, formatted_issues_per_user_per_week, issues_per_user): f = open(base_dir + '{}.txt'.format(project), 'w') print('{0}/{1} Report'.format(organization_name, project), file=f) print('-'*50, end='\n\n', file=f) for i in range(len(weeks)): print('Sprint %d: ' %(i), file=f) print('\tIssues: %d' %(issues_per_week[i]), file=f) print('\tComments: %d' %(comments_per_week[i]), file=f) print('\tComment avg size: %d chars' %(comment_avg_len_per_week[i]), file=f) print('\tTotal comment size: %d chars' %(comments_len_per_week[i]), file=f) print('-'*50, end='\n\n', file=f) print('Total issues: %d' %(sum(issues_per_week)), file=f) print('Total comments: %d' %(sum(comments_per_week)), file=f) f.close() u.save_graph( x_values=weeks, x_label="Weeks", y_values=issues_per_week, y_label="Number of issues", title="Issues open per week", line_label="Number of issues", line_color="b", save_path=base_dir + '{}-issues_per_week.pdf'.format(project) ) u.save_graph( x_values=weeks, x_label="Weeks", y_values=comments_per_week, y_label="Number of comments", title="Comments made per week", line_label="Number of comments", line_color="r", save_path=base_dir + '{}-comments_per_week.pdf'.format(project) ) u.save_graph( x_values=weeks, x_label="Weeks", y_values=comment_avg_len_per_week, y_label="Average comment size", title="Average comment size per week", line_label="Total comments size", line_color="g", save_path=base_dir + '{}-comment_avg_len_per_week.pdf'.format(project) ) u.save_graph( x_values=weeks, x_label="Weeks", y_values=comments_len_per_week, y_label="Total comments size", title="Total comments size per week", line_label="Total comments size", line_color="k", save_path=base_dir + '{}-total_commented_chars_per_week.pdf'.format(project) ) u.save_box_graph( x_values=np.arange(len(issues_per_user)), bar_values=[issues_per_user[author] for author in issues_per_user], bar_labels=issues_per_user, y_label='Number of issues', title='Issues opened per user', save_path=base_dir + '{}-issues_opened_per_user.pdf'.format(project) ) colorset = set() lastint = 0 while(len(colorset) < len(formatted_issues_per_user_per_week)): rand = random.randint(0, 0xFFFFFF) if abs(rand - lastint) < 10: rand += 10 lastint = rand colorset.add("#" + "%06x" % rand) colorset = list(colorset) u.save_multiline_graph( x_values_list=[weeks] * len(formatted_issues_per_user_per_week), x_label="Weeks", y_values_list=[formatted_issues_per_user_per_week[author]['x'] for author in formatted_issues_per_user_per_week], y_label="Number of issues", title="Issues opened per user over time", line_label_list=list(formatted_issues_per_user_per_week), line_color_list=colorset, save_path=base_dir + '{}-issues_opened_per_user_over_time.pdf'.format(project) )
def run( self, n_episodes, level=0, load=False, save_frequency=10, threshold=70, test=True, verbose=False, visualize=True, save_video=True, visualize_directory=None, ): """ ### NO WORK NEEDED ### You can look at the structure but you do not need to modify it. You can print whatever you feel necessary. ###################### Train agent for n_episodes if test == True the agent will not explore and will only exploit. if verbose == True the function will print more information during the training (this will messup the progress bar) if visualize == True the function will display an animation of the rocket landing for every episode. This is at the expense of speed though. If false it will only show it every n episodes. """ self.level_number = level self.env = tensorforce.environments.OpenAIGym("RocketLander-v0", level_number=level) if n_episodes < save_frequency: str_error = f"n_episodes<save frequency, the model won't be able to save, set n_episodes to a value >={save_frenquency}" raise (ValueError(str_error)) agent = self.create_agent(self.env, n_episodes, save_frequency, load) # Loop over episodes reward_list_episodes = [] reward_list = [] score_list = [] landing_fraction_list = [] tqdm_bar = tqdm(range(1, n_episodes + 1)) self.number_of_landings = 0 for i in tqdm_bar: self.fraction_good_landings = self.number_of_landings * 100 / i if i > 1: tqdm_bar.set_description( "Episode %d/%d reward: %d (max:%d, min:%d, mean:%d, std:%d), successful landings:%d(%d%%)" % ( i, n_episodes, np.round(np.sum(reward_list), 3), np.max(reward_list_episodes), np.min(reward_list_episodes), np.round(np.mean(reward_list_episodes), 3), np.round(np.std(reward_list_episodes), 3), self.number_of_landings, np.round(self.fraction_good_landings, 3), )) if i % save_frequency == 0: if save_video: self.env = tensorforce.environments.OpenAIGym( "RocketLander-v0", visualize=True, visualize_directory=visualize_directory, level_number=level, ) else: self.env = tensorforce.environments.OpenAIGym( "RocketLander-v0", visualize=True, level_number=level) reward_list, score = self.episode(self.env, i, agent, test=True, verbose=False) else: self.env = tensorforce.environments.OpenAIGym( "RocketLander-v0", level_number=level) reward_list, score = self.episode(self.env, i, agent, test=False, verbose=False) score_list.append(score) reward_list_episodes.append(np.sum(reward_list)) landing_fraction_list.append(self.fraction_good_landings) if self.env.environment.landed_ticks > 59: self.number_of_landings += 1 if (self.fraction_good_landings > threshold) and (i > 50): print("level cracked") self.cracked = True break # Show Sum of reward over 1 episode vs number of episodes graph reward_list_episodes = save_progress(load, reward_list_episodes, "reward_list_episodes.txt", level=level) score_list = save_progress(load, score_list, "score_list.txt", level=level) landing_fraction_list = save_progress(load, landing_fraction_list, "landing_fraction.txt", level=level) save_graph( reward_list_episodes, "Sum of reward over 1 episode vs number of episodes", "rewards_vs_episodes.png", rolling=True, level=level, ) save_graph( score_list, "Score vs number of episodes", "score_vs_episodes.png", rolling=True, level=level, ) save_graph( landing_fraction_list, "Landing fraction vs number of episodes", "landing_fraction_vs_episodes.png", level=level, ) # SENDING INFO TO DATABASE DATE = str(datetime.datetime.today()) inputs = [np.mean(score_list), n_episodes] result_data = prep_data_to_send(inputs, GROUP_NAME, DATE) send_result(result_data)
import os import networkx as nx import matplotlib.pyplot as plt lib = os.path.join(os.path.abspath('.'), 'lib') sys.path.insert(0, lib) import utils # USER:saicologic user_id = 1502 G = nx.Graph() # 自分が参加したイベントを取得する。 events = utils.get_event_by_user(user_id) # 自分が参加したイベントの人物ネットワークを作成する。 c = 0 for event in events: if c < len(events): utils.add_network(G, event.event_id) c +=1 print '----------------------------------------------------------------------------' # pajeckに出力をする。 utils.save_graph(G, user_id) # グラフを描画する。 nx.draw(G) plt.show()