def gen_25k_graph(O, names=None):
error = 0.05
G = nx.Graph(O)
test25 = Estimation()
gen25 = Generation()
test25.load_graph("", graph=G)
test25.calcfull_CCK()
test25.calcfull_JDD()
gen25.set_JDD(test25.get_JDD('full'))
gen25.set_KTRI(test25.get_KTRI('full'))
gen25.construct_triangles_2K()
gen25.mcmc_improved_2_5_K(error_threshold=error)
H = gen25.G
for i in range(len(O.nodes())-len(H.nodes())):
H.add_node(len(H.nodes()))
if names:
dknames = H.nodes()
mapping = {dknames[i]: names[i] for i in range(len(names))}
H = nx.relabel_nodes(H, mapping)
assert(len(H.nodes()) == len(O.nodes()))
return H
def __init__(self, query1, query2, query3, data_handler):
logger.info("Initializing Score object")
self.data = Data(data_handler)
self.data = self.data.get_clean_datab(query1)
self.estimate = Estimation(query2, data_handler)
self.query3 = query3
self.coeff = {}
self.score = {}
self.score_details = {}
self.coeff = {"PUISSANCE_PERSONNE":1, "TAUX_DE_TRANSFORMATION":1, "PRIX_PERSONNE":1,
"PRIX_METRE":1, "PRIX_PUISSANCE":1, "DELAIS_MOYEN_REPONSE_MESSAGES":1}
self.continuous_variables = ["PUISSANCE_PERSONNE", "TAUX_DE_TRANSFORMATION", "PRIX_PERSONNE",
"PRIX_METRE", "PRIX_PUISSANCE", "DELAIS_MOYEN_REPONSE_MESSAGES"]
self.estimate.get_deviations(self.continuous_variables)
self.estimate.get_means(self.continuous_variables)
logger.info("Score object initialized")
class Score:
def __init__(self, query1, query2, query3, data_handler):
logger.info("Initializing Score object")
self.data = Data(data_handler)
self.data = self.data.get_clean_datab(query1)
self.estimate = Estimation(query2, data_handler)
self.query3 = query3
self.coeff = {}
self.score = {}
self.score_details = {}
self.coeff = {"PUISSANCE_PERSONNE":1, "TAUX_DE_TRANSFORMATION":1, "PRIX_PERSONNE":1,
"PRIX_METRE":1, "PRIX_PUISSANCE":1, "DELAIS_MOYEN_REPONSE_MESSAGES":1}
self.continuous_variables = ["PUISSANCE_PERSONNE", "TAUX_DE_TRANSFORMATION", "PRIX_PERSONNE",
"PRIX_METRE", "PRIX_PUISSANCE", "DELAIS_MOYEN_REPONSE_MESSAGES"]
self.estimate.get_deviations(self.continuous_variables)
self.estimate.get_means(self.continuous_variables)
logger.info("Score object initialized")
def get_score(self):
logger.info("Starting score computation")
row_iterator = self.data.iterrows()
count = 0 # iteration counter for lazy people
doudou = []
for i, row in row_iterator:
count += 1
thread = ComputeZscore(self.estimate, row, i, self.coeff)
thread.start()
dic, ind = thread.join()
self.score[ind] = dic
logger.info("Computed %i scores"%count)
return self.score
def get_coeff(self, data_handler):
table = data_handler.extract_from_table(self.query3, kwargs="ts")
iter = table.iterrows()
for i, row in iter:
self.coeff[row["NOM"]] = row["COEFF"]
@staticmethod
def dict_to_sorted_array(result):
names = ['id', 'data']
formats = ['int32', 'f8']
dtype = dict(names=names, formats=formats)
array = np.fromiter(result.iteritems(), dtype=dtype, count=len(result))
array = np.sort(array, order="data")
return array
@staticmethod
def get_script(result):
count = 0
f = open("C:/Users/Administrateur/Documents/Click&Boat/Thomas/test.bat", "w")
f.write('@echo off\n"C:\Program Files\Google\Chrome\Application\chrome.exe\nsleep 2\n')
for i, j in result:
if count < 20:
f.write('start chrome "https://www.clickandboat.com/en/back-office/3?id=' + str(i) + '"\n')
count += 1
else:
break
f.write(':FIN')
f.close()
def csv_test_unparallel():
eg.__eigen_info = True
print("Strategy," + "Graph," + str("EigErr") + "," + str("DegCorr") + "," +
str("ClustRatio") + "," + str("EigVErr") + "," +
str("NodeDistCorr") + "," + str("DegBetCorr") + "," +
str("KCoreCorr") + "," + str("CommNeighDist") + "," +
str("PartRatio") + "," + str("AvgNeighDegCorr") + "," +
str("Connected"),
file=sys.stderr)
for filo in os.listdir("/home/baldo/tmp/graph_generator/PL200/"):
with open("/home/baldo/tmp/graph_generator/PL200/" + filo, "r") as net:
eg.info(filo)
eg.info("Loading graph..")
G = nx.read_weighted_edgelist(net) # , delimiter=":")
# G = read_graphml(net)
A = nx.to_numpy_matrix(G)
n = nm.shape(A)[0]
joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G)
eg.info("Computing centrality..")
x, l = eg.eigen_centrality(A)
for i in range(10):
eg.info("Run: " + str(i))
eg.info("Building JDM graph..")
H = joint_degree_graph(joint_degrees)
B = nx.to_numpy_matrix(H)
write_statistics(A, B, "2k", filo, x, l)
eg.info("Building degree sequence graph..")
H = nx.random_degree_sequence_graph((nx.degree(G).values()))
B = nx.to_numpy_matrix(H)
write_statistics(A, B, "1k", filo, x, l)
precision = 0.01
eg.info("Building eigen " + str(precision) + " graph..")
B = eg.build_matrix(x, l, precision)
write_statistics(A, B, "eig" + str(precision), filo, x, l)
precision = 0.001
eg.info("Building eigen " + str(precision) + " graph..")
B = eg.build_matrix(x, l, precision)
write_statistics(A, B, "eig" + str(precision), filo, x, l)
precision = 0.0001
eg.info("Building eigen " + str(precision) + " graph..")
B = eg.build_matrix(x, l, precision)
write_statistics(A, B, "eig" + str(precision), filo, x, l)
m = 0.25
eg.info("Building spectral " + str(m) + " graph..")
B = eg.sample_simm_matrix(A, int(round(n * m)))
write_statistics(A, B, "spectre" + str(m), filo, x, l)
m = 0.5
eg.info("Building spectral " + str(m) + " graph..")
B = eg.sample_simm_matrix(A, int(round(n * m)))
write_statistics(A, B, "spectre" + str(m), filo, x, l)
m = 0.75
eg.info("Building spectral " + str(m) + " graph..")
B = eg.sample_simm_matrix(A, int(round(n * m)))
write_statistics(A, B, "spectre" + str(m), filo, x, l)
m = 0.9
eg.info("Building spectral " + str(m) + " graph..")
B = eg.sample_simm_matrix(A, int(round(n * m)))
write_statistics(A, B, "spectre" + str(m), filo, x, l)
m = 0.95
eg.info("Building spectral " + str(m) + " graph..")
B = eg.sample_simm_matrix(A, int(round(n * m)))
write_statistics(A, B, "spectre" + str(m), filo, x, l)
eg.info("Building D2.5 graph..")
test25 = Estimation()
gen25 = Generation()
test25.load_graph("", graph=G)
test25.calcfull_CCK()
test25.calcfull_JDD()
gen25.set_JDD(test25.get_JDD('full'))
gen25.set_KTRI(test25.get_KTRI('full'))
gen25.construct_triangles_2K()
gen25.mcmc_improved_2_5_K(error_threshold=0.05)
H = gen25.G
B = nx.to_numpy_matrix(H)
write_statistics(A, B, "25k", filo, x, l)
def graph_worker_oneshot(inputlist, queue, print_queue):
for duty in inputlist:
name = duty[0]
G = duty[1]
algo = duty[2]
param = duty[3]
A = nx.to_numpy_matrix(G)
# x, l = eg.eigen_centrality(A)
eg.info("Setup completed")
start_time = time.time()
if algo == "1k":
H = nx.random_degree_sequence_graph((nx.degree(G).values()))
# B = nx.to_numpy_matrix(H)
elif algo == "2k":
joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G)
H = joint_degree_graph(joint_degrees)
# B = nx.to_numpy_matrix(H)
elif algo == "eig":
precision = float(param)
# B = eg.build_matrix(x, l, precision)
B = eg.generate_matrix(x, l * x, precision, gu.get_degrees(A))
H = None
algo += str(precision)
elif algo == "modeig":
precision = float(param)
B = eg.synthetic_modularity_matrix(A, precision)
H = None
algo += str(precision)
elif algo == "spectre":
m = float(param)
n = nm.shape(A)[0]
B = eg.sample_simm_matrix2(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
while nx.is_isomorphic(G, H):
B = eg.sample_simm_matrix2(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
algo += str(m)
elif algo == "laplacian":
m = float(param)
n = nm.shape(A)[0]
B = eg.laplacian_clone_matrix(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
while nx.is_isomorphic(G, H):
B = eg.sample_simm_matrix2(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
algo += str(m)
elif algo == "modspec":
m = float(param)
n = nm.shape(A)[0]
B = eg.modspec_clone_matrix(A, int(round(n * m)))
H = None
algo += str(m)
elif algo == "franky":
m = float(param)
n = nm.shape(A)[0]
B = eg.franky_clone_matrix(A, int(round(n * m)))
H = None
algo += str(m)
elif algo == "modularity":
m = float(param)
n = nm.shape(A)[0]
B = eg.modularity_clone_matrix(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
while nx.is_isomorphic(G, H):
B = eg.modularity_clone_matrix(A, int(round(n * m)))
H = gu.simm_matrix_2_graph(B)
algo += str(m)
elif algo == "25k":
test25 = Estimation()
gen25 = Generation()
test25.load_graph("", graph=G)
test25.calcfull_CCK()
test25.calcfull_JDD()
gen25.set_JDD(test25.get_JDD('full'))
gen25.set_KTRI(test25.get_KTRI('full'))
gen25.construct_triangles_2K()
gen25.mcmc_improved_2_5_K(error_threshold=0.05)
H = gen25.G
# B = nx.to_numpy_matrix(H)
eg.info("Graph Generated")
stat = get_statistics1(G, H, time.time() - start_time)
s = algo + "," + name + "," + str(len(G.nodes()))
for el in stat:
s += "," + str(el)
print_queue.put(s)
print_queue.put("\n")
gc.collect()
def graph_worker(inputlist, queue, print_queue):
for filo in inputlist:
if filo.split(".")[-1] == "graphml":
G = read_graphml(filo)
else:
G = nx.read_weighted_edgelist(filo)
A = nx.to_numpy_matrix(G)
n = nm.shape(A)[0]
joint_degrees = nx.algorithms.mixing.degree_mixing_dict(G)
x, l = eg.eigen_centrality(A)
H = nx.random_degree_sequence_graph((nx.degree(G).values()))
B = nx.to_numpy_matrix(H)
print_queue.put(write_statistics(A, B, "1k", filo, x, l, output=False))
print_queue.put("\n")
H = joint_degree_graph(joint_degrees)
B = nx.to_numpy_matrix(H)
print_queue.put(write_statistics(A, B, "2k", filo, x, l, output=False))
print_queue.put("\n")
precision = 0.01
B = eg.build_matrix(x, l, precision)
print_queue.put(
write_statistics(A,
B,
"eig" + str(precision),
filo,
x,
l,
output=False))
print_queue.put("\n")
precision = 0.001
B = eg.build_matrix(x, l, precision)
print_queue.put(
write_statistics(A,
B,
"eig" + str(precision),
filo,
x,
l,
output=False))
print_queue.put("\n")
precision = 0.0001
B = eg.build_matrix(x, l, precision)
print_queue.put(
write_statistics(A,
B,
"eig" + str(precision),
filo,
x,
l,
output=False))
print_queue.put("\n")
m = 0.25
B = eg.sample_simm_matrix(A, int(round(n * m)))
print_queue.put(
write_statistics(A,
B,
"spectre" + str(m),
filo,
x,
l,
output=False))
print_queue.put("\n")
m = 0.5
B = eg.sample_simm_matrix(A, int(round(n * m)))
print_queue.put(
write_statistics(A,
B,
"spectre" + str(m),
filo,
x,
l,
output=False))
print_queue.put("\n")
m = 0.75
B = eg.sample_simm_matrix(A, int(round(n * m)))
print_queue.put(
write_statistics(A,
B,
"spectre" + str(m),
filo,
x,
l,
output=False))
print_queue.put("\n")
m = 0.9
B = eg.sample_simm_matrix(A, int(round(n * m)))
print_queue.put(
write_statistics(A,
B,
"spectre" + str(m),
filo,
x,
l,
output=False))
print_queue.put("\n")
m = 0.95
B = eg.sample_simm_matrix(A, int(round(n * m)))
print_queue.put(
write_statistics(A,
B,
"spectre" + str(m),
filo,
x,
l,
output=False))
print_queue.put("\n")
test25 = Estimation()
gen25 = Generation()
test25.load_graph("", graph=G)
test25.calcfull_CCK()
test25.calcfull_JDD()
gen25.set_JDD(test25.get_JDD('full'))
gen25.set_KTRI(test25.get_KTRI('full'))
gen25.construct_triangles_2K()
gen25.mcmc_improved_2_5_K(error_threshold=0.05)
H = gen25.G
B = nx.to_numpy_matrix(H)
print_queue.put(write_statistics(A, B, "25k", filo, x, l,
output=False))
print_queue.put("\n")
def quick_test_estimation_aberrations():
# Import packages
#################
import os
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
from datetime import date
from datetime import datetime
from astropy.io import fits
from Simulator import Simulation
from Estimation import Estimation
# Set up directories
####################
today = date.today()
date = test_date = today.strftime("%Y%m%d")
print(test_date)
tests_directory = './Tests/' + test_date + '/'
if not os.path.exists(tests_directory):
os.makedirs(tests_directory)
directory = tests_directory + test_date + '_quick_test_estimation_aberrations/'
if not os.path.exists(directory):
os.makedirs(directory)
# Parameters simulation images
##############################
#transmission = '/Users/mygouf/Python/webbpsf/webbpsf-data4/jwst_pupil_RevW_npix1024.fits.gz'
#opd = '/Users/mygouf/Python/webbpsf/webbpsf-data4/NIRCam/OPD/OPD_RevW_ote_for_NIRCam_requirements.fits.gz'
wfe_budget = [
0, 2000, 2000, 1500, 1000, 1000, 500, 360, 360, 250, 100, 100, 80, 70,
70, 60, 50, 50, 40, 30, 30, 20, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4, 4, 3,
2, 2, 1
]
#wfe_budget = [0, 2000, 2000] #, 1500, 1000, 1000, 500, 360, 360, 250, 100, 100, 80, 70, 70, 60, 50, 50, 40, 30, 30, 20, 10, 10, 9, 8, 8, 7, 6, 6, 5, 4, 4, 3, 2, 2, 1]
# poppy paramaters
#pixelscale = 0.063
fov_arcsec = 10
#oversample = 4
#wavelength = 4.441e-6
# webbPSF parameters
#filt = 'F444W'
# Simulation PSF
################
dict_simulation_parameters = {
'wfe_budget': wfe_budget,
'fov_arcsec': fov_arcsec
}
simulation = Simulation(dict_simulation_parameters)
dict_, coefficient_set_init = simulation.generate_psfs()
image = dict_['image']
noise = dict_['noise']
wavefront_map = dict_['wavefront_map']
dict_['wfe_budget'] = wfe_budget
dict_['fov_arcsec'] = fov_arcsec
image_simu = image.copy()
noise_simu = noise.copy()
wavefront_map_simu = wavefront_map.copy()
coefficient_set_init_simu = coefficient_set_init.copy()
# Estimation Zernike coefficients - without noise same budget
#################################
method = 'L-BFGS-B'
test = '_without_noise_same_budget'
wfe_budget_estimation = wfe_budget
dict_['wfe_budget'] = wfe_budget_estimation
dict_['noise'] = noise_simu
now = datetime.now()
#estimation = Estimation(dict_,guess=coefficient_set_init_simu[0])
estimation = Estimation(dict_)
final_image, estimated_coefficients = estimation.estimate_zernikes(
method=method)
print('Simulated coefficients', coefficient_set_init)
dict_estimated = simulation.create_image(estimated_coefficients[0])
wavefront_map_estimated = dict_estimated['wavefront_map']
now2 = datetime.now()
print("Time to estimate", now2 - now)
# Save fits files
write_fits(directory + date + test + '_image_simulation.fits', image_simu)
write_fits(directory + date + test + '_noise_simulation.fits', noise_simu)
write_fits(
directory + date + test + '_Zernike_coefficients_simulation.fits',
coefficient_set_init_simu)
write_fits(directory + date + test + '_wavefront_map_simulation.fits',
wavefront_map_simu)
write_fits(directory + date + test + '_image_estimation.fits', final_image)
write_fits(
directory + date + test + '_Zernike_coefficients_estimation.fits',
estimated_coefficients[0])
write_fits(directory + date + test + '_wavefront_map_estimation.fits',
wavefront_map_estimated)
# Create and saving figures
figure_images(image_simu, final_image, directory + date + test)
figure_wavefronts(wavefront_map_simu, wavefront_map_estimated,
directory + date + test)
figure_coefficients(coefficient_set_init_simu, estimated_coefficients,
directory + date + test)
'''
You can find below topologies at
http://www.minasgjoka.com/2.5K
'''
fname = "UCSD34.mat"
#fname = "Harvard1.mat"
#fname = "Facebook-New-Orleans.edges.gz"
#fname = 'soc-Epinions1.edges.gz'
#fname = "email-Enron.edges.gz"
#fname = "as-caida20071105.edges.gz"
run_case = 1
###### Full graph - 2K with triangles + Improved MCMC
if run_case == 1:
myest = Estimation()
myest.load_graph(fname)
myest.calcfull_CCK()
myest.calcfull_JDD()
mygen = Generation()
mygen.set_JDD( myest.get_JDD('full') )
mygen.set_KTRI( myest.get_KTRI('full') )
mygen.construct_triangles_2K()
mygen.mcmc_improved_2_5_K(nmae_threshold=0.03)
mygen.save_graphs('%s_2KT+ImpMCMC_Full' % fname)
#######################################################
###### Full graph - 2K simple + MCMC
elif run_case == 2:
myest = Estimation()
#fname = "Facebook-New-Orleans.edges.gz"
#fname = 'soc-Epinions1.edges.gz'
#fname = "email-Enron.edges.gz"
#fname = "as-caida20071105.edges.gz"
#fname = "out.ca-AstroPh.edges.gz"
fname = "powerlaw_cluster" # from networkx.generators.powerlaw_cluster_graph ; n=5000
#fname = "web-NotreDame.edges.gz"
#fname = "web-Google.edges.gz"
#fname = "wiki-Talk.edges.gz"
run_case = 1
error_threshold = 0.05
###### Full graph - 2K with triangles + Improved MCMC
if run_case == 1:
myest = Estimation()
myest.load_graph(fname)
myest.calcfull_CCK()
myest.calcfull_JDD()
myest.estimation_summary()
mygen = Generation()
mygen.set_JDD(myest.get_JDD('full'))
mygen.set_KTRI(myest.get_KTRI('full'))
mygen.construct_triangles_2K()
mygen.mcmc_improved_2_5_K(error_threshold=error_threshold)
mygen.save_graphs('%s_2KT+ImpMCMC_Full' % fname)
#######################################################
###### Full graph - 2K simple + MCMC
