def gen_25k_graph(O, names=None): error = 0.05 G = nx.Graph(O) test25 = Estimation() gen25 = Generation() test25.load_graph("", graph=G) test25.calcfull_CCK() test25.calcfull_JDD() gen25.set_JDD(test25.get_JDD('full')) gen25.set_KTRI(test25.get_KTRI('full')) gen25.construct_triangles_2K() gen25.mcmc_improved_2_5_K(error_threshold=error) H = gen25.G for i in range(len(O.nodes())-len(H.nodes())): H.add_node(len(H.nodes())) if names: dknames = H.nodes() mapping = {dknames[i]: names[i] for i in range(len(names))} H = nx.relabel_nodes(H, mapping) assert(len(H.nodes()) == len(O.nodes())) return H
def csv_test_unparallel(): eg.__eigen_info = True print("Strategy," + "Graph," + str("EigErr") + "," + str("DegCorr") + "," + str("ClustRatio") + "," + str("EigVErr") + "," + str("NodeDistCorr") + "," + str("DegBetCorr") + "," + str("KCoreCorr") + "," + str("CommNeighDist") + "," + str("PartRatio") + "," + str("AvgNeighDegCorr") + "," + str("Connected"), file=sys.stderr) for filo in os.listdir("/home/baldo/tmp/graph_generator/PL200/"): with open("/home/baldo/tmp/graph_generator/PL200/" + filo, "r") as net: eg.info(filo) eg.info("Loading graph..") G = nx.read_weighted_edgelist(net) # , delimiter=":") # G = read_graphml(net) A = nx.to_numpy_matrix(G) n = nm.shape(A)[0] joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G) eg.info("Computing centrality..") x, l = eg.eigen_centrality(A) for i in range(10): eg.info("Run: " + str(i)) eg.info("Building JDM graph..") H = joint_degree_graph(joint_degrees) B = nx.to_numpy_matrix(H) write_statistics(A, B, "2k", filo, x, l) eg.info("Building degree sequence graph..") H = nx.random_degree_sequence_graph((nx.degree(G).values())) B = nx.to_numpy_matrix(H) write_statistics(A, B, "1k", filo, x, l) precision = 0.01 eg.info("Building eigen " + str(precision) + " graph..") B = eg.build_matrix(x, l, precision) write_statistics(A, B, "eig" + str(precision), filo, x, l) precision = 0.001 eg.info("Building eigen " + str(precision) + " graph..") B = eg.build_matrix(x, l, precision) write_statistics(A, B, "eig" + str(precision), filo, x, l) precision = 0.0001 eg.info("Building eigen " + str(precision) + " graph..") B = eg.build_matrix(x, l, precision) write_statistics(A, B, "eig" + str(precision), filo, x, l) m = 0.25 eg.info("Building spectral " + str(m) + " graph..") B = eg.sample_simm_matrix(A, int(round(n * m))) write_statistics(A, B, "spectre" + str(m), filo, x, l) m = 0.5 eg.info("Building spectral " + str(m) + " graph..") B = eg.sample_simm_matrix(A, int(round(n * m))) write_statistics(A, B, "spectre" + str(m), filo, x, l) m = 0.75 eg.info("Building spectral " + str(m) + " graph..") B = eg.sample_simm_matrix(A, int(round(n * m))) write_statistics(A, B, "spectre" + str(m), filo, x, l) m = 0.9 eg.info("Building spectral " + str(m) + " graph..") B = eg.sample_simm_matrix(A, int(round(n * m))) write_statistics(A, B, "spectre" + str(m), filo, x, l) m = 0.95 eg.info("Building spectral " + str(m) + " graph..") B = eg.sample_simm_matrix(A, int(round(n * m))) write_statistics(A, B, "spectre" + str(m), filo, x, l) eg.info("Building D2.5 graph..") test25 = Estimation() gen25 = Generation() test25.load_graph("", graph=G) test25.calcfull_CCK() test25.calcfull_JDD() gen25.set_JDD(test25.get_JDD('full')) gen25.set_KTRI(test25.get_KTRI('full')) gen25.construct_triangles_2K() gen25.mcmc_improved_2_5_K(error_threshold=0.05) H = gen25.G B = nx.to_numpy_matrix(H) write_statistics(A, B, "25k", filo, x, l)
def graph_worker_oneshot(inputlist, queue, print_queue): for duty in inputlist: name = duty[0] G = duty[1] algo = duty[2] param = duty[3] A = nx.to_numpy_matrix(G) # x, l = eg.eigen_centrality(A) eg.info("Setup completed") start_time = time.time() if algo == "1k": H = nx.random_degree_sequence_graph((nx.degree(G).values())) # B = nx.to_numpy_matrix(H) elif algo == "2k": joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G) H = joint_degree_graph(joint_degrees) # B = nx.to_numpy_matrix(H) elif algo == "eig": precision = float(param) # B = eg.build_matrix(x, l, precision) B = eg.generate_matrix(x, l * x, precision, gu.get_degrees(A)) H = None algo += str(precision) elif algo == "modeig": precision = float(param) B = eg.synthetic_modularity_matrix(A, precision) H = None algo += str(precision) elif algo == "spectre": m = float(param) n = nm.shape(A)[0] B = eg.sample_simm_matrix2(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) while nx.is_isomorphic(G, H): B = eg.sample_simm_matrix2(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) algo += str(m) elif algo == "laplacian": m = float(param) n = nm.shape(A)[0] B = eg.laplacian_clone_matrix(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) while nx.is_isomorphic(G, H): B = eg.sample_simm_matrix2(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) algo += str(m) elif algo == "modspec": m = float(param) n = nm.shape(A)[0] B = eg.modspec_clone_matrix(A, int(round(n * m))) H = None algo += str(m) elif algo == "franky": m = float(param) n = nm.shape(A)[0] B = eg.franky_clone_matrix(A, int(round(n * m))) H = None algo += str(m) elif algo == "modularity": m = float(param) n = nm.shape(A)[0] B = eg.modularity_clone_matrix(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) while nx.is_isomorphic(G, H): B = eg.modularity_clone_matrix(A, int(round(n * m))) H = gu.simm_matrix_2_graph(B) algo += str(m) elif algo == "25k": test25 = Estimation() gen25 = Generation() test25.load_graph("", graph=G) test25.calcfull_CCK() test25.calcfull_JDD() gen25.set_JDD(test25.get_JDD('full')) gen25.set_KTRI(test25.get_KTRI('full')) gen25.construct_triangles_2K() gen25.mcmc_improved_2_5_K(error_threshold=0.05) H = gen25.G # B = nx.to_numpy_matrix(H) eg.info("Graph Generated") stat = get_statistics1(G, H, time.time() - start_time) s = algo + "," + name + "," + str(len(G.nodes())) for el in stat: s += "," + str(el) print_queue.put(s) print_queue.put("\n") gc.collect()
def graph_worker(inputlist, queue, print_queue): for filo in inputlist: if filo.split(".")[-1] == "graphml": G = read_graphml(filo) else: G = nx.read_weighted_edgelist(filo) A = nx.to_numpy_matrix(G) n = nm.shape(A)[0] joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G) x, l = eg.eigen_centrality(A) H = nx.random_degree_sequence_graph((nx.degree(G).values())) B = nx.to_numpy_matrix(H) print_queue.put(write_statistics(A, B, "1k", filo, x, l, output=False)) print_queue.put("\n") H = joint_degree_graph(joint_degrees) B = nx.to_numpy_matrix(H) print_queue.put(write_statistics(A, B, "2k", filo, x, l, output=False)) print_queue.put("\n") precision = 0.01 B = eg.build_matrix(x, l, precision) print_queue.put( write_statistics(A, B, "eig" + str(precision), filo, x, l, output=False)) print_queue.put("\n") precision = 0.001 B = eg.build_matrix(x, l, precision) print_queue.put( write_statistics(A, B, "eig" + str(precision), filo, x, l, output=False)) print_queue.put("\n") precision = 0.0001 B = eg.build_matrix(x, l, precision) print_queue.put( write_statistics(A, B, "eig" + str(precision), filo, x, l, output=False)) print_queue.put("\n") m = 0.25 B = eg.sample_simm_matrix(A, int(round(n * m))) print_queue.put( write_statistics(A, B, "spectre" + str(m), filo, x, l, output=False)) print_queue.put("\n") m = 0.5 B = eg.sample_simm_matrix(A, int(round(n * m))) print_queue.put( write_statistics(A, B, "spectre" + str(m), filo, x, l, output=False)) print_queue.put("\n") m = 0.75 B = eg.sample_simm_matrix(A, int(round(n * m))) print_queue.put( write_statistics(A, B, "spectre" + str(m), filo, x, l, output=False)) print_queue.put("\n") m = 0.9 B = eg.sample_simm_matrix(A, int(round(n * m))) print_queue.put( write_statistics(A, B, "spectre" + str(m), filo, x, l, output=False)) print_queue.put("\n") m = 0.95 B = eg.sample_simm_matrix(A, int(round(n * m))) print_queue.put( write_statistics(A, B, "spectre" + str(m), filo, x, l, output=False)) print_queue.put("\n") test25 = Estimation() gen25 = Generation() test25.load_graph("", graph=G) test25.calcfull_CCK() test25.calcfull_JDD() gen25.set_JDD(test25.get_JDD('full')) gen25.set_KTRI(test25.get_KTRI('full')) gen25.construct_triangles_2K() gen25.mcmc_improved_2_5_K(error_threshold=0.05) H = gen25.G B = nx.to_numpy_matrix(H) print_queue.put(write_statistics(A, B, "25k", filo, x, l, output=False)) print_queue.put("\n")
def quick_test_estimation_aberrations(): # Import packages ################# import os import numpy as np import matplotlib.pyplot as plt from matplotlib.colors import LogNorm from datetime import date from datetime import datetime from astropy.io import fits from Simulator import Simulation from Estimation import Estimation # Set up directories #################### today = date.today() date = test_date = today.strftime("%Y%m%d") print(test_date) tests_directory = './Tests/' + test_date + '/' if not os.path.exists(tests_directory): os.makedirs(tests_directory) directory = tests_directory + test_date + '_quick_test_estimation_aberrations/' if not os.path.exists(directory): os.makedirs(directory) # Parameters simulation images ############################## #transmission = '/Users/mygouf/Python/webbpsf/webbpsf-data4/jwst_pupil_RevW_npix1024.fits.gz' #opd = '/Users/mygouf/Python/webbpsf/webbpsf-data4/NIRCam/OPD/OPD_RevW_ote_for_NIRCam_requirements.fits.gz' wfe_budget = [ 0, 2000, 2000, 1500, 1000, 1000, 500, 360, 360, 250, 100, 100, 80, 70, 70, 60, 50, 50, 40, 30, 30, 20, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4, 4, 3, 2, 2, 1 ] #wfe_budget = [0, 2000, 2000] #, 1500, 1000, 1000, 500, 360, 360, 250, 100, 100, 80, 70, 70, 60, 50, 50, 40, 30, 30, 20, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4, 4, 3, 2, 2, 1] # poppy paramaters #pixelscale = 0.063 fov_arcsec = 10 #oversample = 4 #wavelength = 4.441e-6 # webbPSF parameters #filt = 'F444W' # Simulation PSF ################ dict_simulation_parameters = { 'wfe_budget': wfe_budget, 'fov_arcsec': fov_arcsec } simulation = Simulation(dict_simulation_parameters) dict_, coefficient_set_init = simulation.generate_psfs() image = dict_['image'] noise = dict_['noise'] wavefront_map = dict_['wavefront_map'] dict_['wfe_budget'] = wfe_budget dict_['fov_arcsec'] = fov_arcsec image_simu = image.copy() noise_simu = noise.copy() wavefront_map_simu = wavefront_map.copy() coefficient_set_init_simu = coefficient_set_init.copy() # Estimation Zernike coefficients - without noise same budget ################################# method = 'L-BFGS-B' test = '_without_noise_same_budget' wfe_budget_estimation = wfe_budget dict_['wfe_budget'] = wfe_budget_estimation dict_['noise'] = noise_simu now = datetime.now() #estimation = Estimation(dict_,guess=coefficient_set_init_simu[0]) estimation = Estimation(dict_) final_image, estimated_coefficients = estimation.estimate_zernikes( method=method) print('Simulated coefficients', coefficient_set_init) dict_estimated = simulation.create_image(estimated_coefficients[0]) wavefront_map_estimated = dict_estimated['wavefront_map'] now2 = datetime.now() print("Time to estimate", now2 - now) # Save fits files write_fits(directory + date + test + '_image_simulation.fits', image_simu) write_fits(directory + date + test + '_noise_simulation.fits', noise_simu) write_fits( directory + date + test + '_Zernike_coefficients_simulation.fits', coefficient_set_init_simu) write_fits(directory + date + test + '_wavefront_map_simulation.fits', wavefront_map_simu) write_fits(directory + date + test + '_image_estimation.fits', final_image) write_fits( directory + date + test + '_Zernike_coefficients_estimation.fits', estimated_coefficients[0]) write_fits(directory + date + test + '_wavefront_map_estimation.fits', wavefront_map_estimated) # Create and saving figures figure_images(image_simu, final_image, directory + date + test) figure_wavefronts(wavefront_map_simu, wavefront_map_estimated, directory + date + test) figure_coefficients(coefficient_set_init_simu, estimated_coefficients, directory + date + test)
#fname = "Facebook-New-Orleans.edges.gz" #fname = 'soc-Epinions1.edges.gz' #fname = "email-Enron.edges.gz" #fname = "as-caida20071105.edges.gz" #fname = "out.ca-AstroPh.edges.gz" fname = "powerlaw_cluster" # from networkx.generators.powerlaw_cluster_graph ; n=5000 #fname = "web-NotreDame.edges.gz" #fname = "web-Google.edges.gz" #fname = "wiki-Talk.edges.gz" run_case = 1 error_threshold = 0.05 ###### Full graph - 2K with triangles + Improved MCMC if run_case == 1: myest = Estimation() myest.load_graph(fname) myest.calcfull_CCK() myest.calcfull_JDD() myest.estimation_summary() mygen = Generation() mygen.set_JDD(myest.get_JDD('full')) mygen.set_KTRI(myest.get_KTRI('full')) mygen.construct_triangles_2K() mygen.mcmc_improved_2_5_K(error_threshold=error_threshold) mygen.save_graphs('%s_2KT+ImpMCMC_Full' % fname) ####################################################### ###### Full graph - 2K simple + MCMC