def ged(g1, g2, algo, debug=False, timeit=False):
# https://github.com/dan-zam/graph-matching-toolkit
gp = get_gmt_path()
append_str = get_append_str(g1, g2)
src, t_datapath = setup_temp_data_folder(gp, append_str)
meta1 = write_to_temp(g1, t_datapath, algo, 'g1')
meta2 = write_to_temp(g2, t_datapath, algo, 'g2')
if meta1 != meta2:
raise RuntimeError('Different meta data {} vs {}'.format(meta1, meta2))
prop_file = setup_property_file(src, gp, meta1, append_str)
rtn = []
if not exec(
'cd {} && java {}'
' -classpath {}/src/graph-matching-toolkit/bin algorithms.GraphMatching '
'./properties/properties_temp_{}.prop'.format(
gp, '-XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -Xmx50g'
if algo == 'astar' else '', get_root_path(), append_str)):
rtn.append(-1)
else:
d, t, lcnt, g1size, g2size, result_file = get_result(
gp, algo, append_str)
rtn.append(d)
if g1size != g1.number_of_nodes():
print('g1size {} g1.number_of_nodes() {}'.format(
g1size, g1.number_of_nodes()))
assert (g1size == g1.number_of_nodes())
assert (g2size == g2.number_of_nodes())
if debug:
rtn += [lcnt, g1, g2]
if timeit:
rtn.append(t)
clean_up(t_datapath, prop_file, result_file)
if len(rtn) == 1:
return rtn[0]
return tuple(rtn)
def __init__(self, import_name, defaults=None):
dict.__init__(self, defaults or {})
self.root_path = get_root_path(import_name)
self.from_pyfile('settings.py')
profile_active = self.get('PROFILE_ACTIVE')
if profile_active is not None:
self.from_pyfile('settings-%s.py' % profile_active)
def exp7():
dataset = 'aids10k'
model = 'vj'
train_data = load_data(dataset, True)
test_data = load_data(dataset, False)
m = len(test_data.graphs)
n = len(train_data.graphs)
ged_mat = np.zeros((m, n))
time_mat = np.zeros((n, n))
outdir = get_root_path() + '/files'
file = open('{}/ged_{}_{}_{}.csv'.format( \
outdir, dataset, model, get_ts()), 'w')
print_and_log('i,j,i_node,j_node,i_edge,j_edge,ged,time', file)
for i in range(m):
for j in range(n):
g1 = test_data.graphs[i]
g2 = train_data.graphs[j]
t = time()
d = ged(g1, g2, model)
t = time() - t
s = '{},{},{},{},{},{},{},{:.5f}'.format(i, j, \
g1.number_of_nodes(),
g2.number_of_nodes(), \
g1.number_of_edges(),
g2.number_of_edges(), \
d, t)
print_and_log(s, file)
ged_mat[i][j] = d
time_mat[i][j] = t
file.close()
np.save('{}/ged_ged_mat_{}_{}_{}'.format( \
outdir, dataset, model, get_ts()), ged_mat)
np.save('{}/ged_time_mat_{}_{}_{}'.format( \
outdir, dataset, model, get_ts()), time_mat)
def exp8_helper(dataset, models, metric, rs):
font = {'family': 'serif', 'size': 22}
matplotlib.rc('font', **font)
plt.figure(0)
plt.figure(figsize=(16, 10))
xs = get_test_graph_sizes(dataset)
so = np.argsort(xs)
xs.sort()
for model in models:
mat = rs[model].mat(metric.name)
print('plotting for {}'.format(model))
ys = np.mean(mat, 1)[so]
plt.plot(xs, ys, **args1[model])
plt.scatter(xs, ys, s=200, label=model, **args2[model])
plt.xlabel('query graph size')
ax = plt.gca()
ax.set_xticks(xs)
plt.ylabel('average {}'.format(metric.ylabel))
plt.legend(loc='best', ncol=2)
plt.grid(linestyle='dashed')
plt.tight_layout()
# plt.show()
plt.savefig(get_root_path() + '/files/{}/{}/ged_{}_mat_{}_{}.png'.format( \
dataset, metric, metric, dataset, '_'.join(models)))
def gen_aids10k():
datadir = get_root_path() + '/data'
dirin = datadir + '/AIDS'
graphs = {}
nodes_graphs = defaultdict(list)
lesseq30 = set()
disconnects = set()
for file in glob(dirin + '/*.gexf'):
gid = int(file.split('/')[-1].split('.')[0])
g = nx.read_gexf(file)
if not nx.is_connected(g):
print('{} not connected'.format(gid))
disconnects.add(gid)
continue
graphs[gid] = g
nodes_graphs[g.number_of_nodes()].append(gid)
if g.number_of_nodes() <= 30:
lesseq30.add(gid)
print(len(disconnects), disconnects)
# exit(1)
# print(nodes_graphs[222])
# print(nodes_graphs[2])
train_dir = '{}/AIDS10k/train'.format(datadir)
test_dir = '{}/AIDS10k/test'.format(datadir)
exec('mkdir -p {}'.format(train_dir))
exec('mkdir -p {}'.format(test_dir))
for num_node in range(5, 23):
choose = sample(nodes_graphs[num_node], 1)[0]
print('choose {} with {} nodes'.format(choose, num_node))
nx.write_gexf(graphs[choose], test_dir + '/{}.gexf'.format(choose))
lesseq30.remove(choose)
for tid in sample(lesseq30, 10000):
nx.write_gexf(graphs[tid], train_dir + '/{}.gexf'.format(tid))
print('Done')
def main():
root = utils.get_root_path(False)
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
parser.add_option('--learning_rate_rbm',
action='store',
type='string',
dest='learning_rate_rbm')
parser.add_option('--epochs_rbm',
action='store',
type='string',
dest='epochs_rbm')
parser.add_option('--batch_size',
action='store',
type='string',
dest='batch_size')
parser.add_option('--data_set',
action='store',
type='string',
dest='data_set')
(opts, args) = parser.parse_args()
file_data = ReadFile.ReadFile(root + '/NSL_KDD-master',
opts=opts).get_data()
data_pp = preprocess.Preprocess(file_data).do_predict_preprocess()
dbn_model.DBN(data_pp).do_dbn('yadlt', opts=opts)
dbn_model.DBN(data_pp).do_dbn_with_weight_matrix(root + '/save')
model.do_svm()
def get_contacts_for_country(variables):
f = open(get_root_path() + '/data/contact_matrix.csv', 'r')
max_age = variables['max_age']
df = pd.read_csv(f, header=0)
df = df[df.country == variables['country']].drop(columns='country')
df = df[df.type == 'all'].drop(columns='type')
df = df.set_index(['age of contact'])
df.columns.name = 'age group of participant'
df = df.sum(axis=0)
ages = []
counts = []
for age_group, count in df.items():
if age_group == '70+':
start, end = 70, variables['max_age']
else:
start, end = map(int, age_group.split('-'))
for age in range(start, end + 1):
ages.append(age)
counts.append(count)
return pd.Series(counts, index=ages)
def get_detected_cases(variables):
area_name = variables['area_name']
assert area_name == 'HUS'
f = open(CASES_FNAME, 'r')
df = pd.read_csv(f, header=0)
df['date'] = pd.to_datetime(df['date']).dt.date
df = df.set_index('date')
return df
cdf = df[['district',
'confirmed']].reset_index().set_index(['date', 'district'
]).unstack('district')
cdf['total'] = cdf.sum(axis=1)
ratio = cdf[('confirmed', area_name)] / cdf['total']
ratio = ratio.iloc[-1]
df = df[df.district == area_name].drop(columns='district')
f = open(get_root_path() + '/data/hospitalizations_fin.csv', 'r')
hdf = pd.read_csv(f, header=0).set_index('date')
hdf = (hdf * ratio).dropna().astype(int)
df['hospitalized'] = hdf['hospitalized']
df['in_icu'] = hdf['in_icu']
df = df.dropna()
return df
def exp5():
font = {'family': 'serif', 'size': 22}
matplotlib.rc('font', **font)
file = 'ged_astar_beam5_beam10_beam20_beam40_beam80_hungarian_vj_2018-04-29T14:56:38.676491'
data = read_csv(get_root_path() + '/files/{}.csv'.format(file))
models = []
for model in data.columns.values:
if 'time' in model:
models.append(model.split('_')[1])
if 'astar' in models:
for i, t in data['time_astar'].iteritems():
if t >= 300:
data.loc[i, 'time_astar'] = 300
data.loc[i, 'ged_astar'] = 0
print(data)
plt.figure(0)
plt.figure(figsize=(16, 10))
for i, model in enumerate(models):
if model == 'astar':
continue
plt.scatter(data['g2_node'],
data['time_' + model],
s=150,
label=model,
**args2[model])
plt.xlabel('# nodes of graph 2')
plt.ylabel('time (sec)')
plt.legend(loc='best')
plt.grid(linestyle='dashed')
plt.tight_layout()
plt.savefig(get_root_path() + '/files/{}_time.png'.format(file))
# plt.show()
plt.figure(1)
plt.figure(figsize=(11, 11))
for i, model in enumerate(models):
plt.scatter(data['ged_astar'],
data['ged_' + model],
s=150,
label=model,
**args[i])
plt.xlabel('true ged')
plt.ylabel('ged')
plt.xlim(1, 57)
plt.legend(loc='best')
plt.grid(linestyle='dashed')
plt.tight_layout()
plt.savefig(get_root_path() + '/files/{}_ged.png'.format(file))
def _get_result_mat(self, dataset, metric):
file_p = get_root_path() + '/files/{}/{}/ged_{}_mat_{}_{}_*.npy'.format( \
dataset, metric, metric, dataset, self.model_)
li = glob(file_p)
if len(li) != 1:
raise RuntimeError('Files for {}: {}'.format(file_p, li))
file = li[0]
return np.load(file)
def setup_temp_data_folder(gp, append_str):
dir = gp + '/data'
create_dir_if_not_exists(dir)
tp = dir + '/temp_{}'.format(append_str)
exec_cmd('rm -rf {} && mkdir {}'.format(tp, tp))
src = get_root_path() + '/src/gmt_files'
exec_cmd('cp {}/temp.xml {}/temp_{}.xml'.format(src, tp, append_str))
return src, tp
def __init__(self):
model_str = self.get_model_str()
self.logdir = join(get_root_path(), 'logs',
'{}_{}'.format(model_str, get_ts()))
create_dir_if_not_exists(self.logdir)
self.model_info_f = self._open('model_info.txt')
self._log_model_info()
self._save_conf_code()
print('Logging to {}'.format(self.logdir))
def get_model_info_as_command():
rtn = []
d = vars(FLAGS)
for k in sorted_nicely(d.keys()):
v = d[k]
s = '--{}={}'.format(k, v)
rtn.append(s)
return 'python {} {}'.format(join(get_root_path(), 'main.py'),
' '.join(rtn))
def init(self):
self.graphs = []
datadir = get_root_path() + '/data/AIDS10k/' + ('train' if self.train \
else 'test')
for file in glob(datadir + '/*.gexf'):
gid = int(file.split('/')[-1].split('.')[0])
g = nx.read_gexf(file)
g.graph['gid'] = gid
self.graphs.append(g)
if not nx.is_connected(g):
raise RuntimeError('{} not connected'.format(gid))
print('Loaded {} graphs from {}'.format(len(self.graphs), datadir))
def init_logging():
logger = logging.getLogger()
s_handler = logging.StreamHandler()
f_handler = logging.FileHandler(filename=os.path.join(utils.get_root_path(), 'logs', 'greenHouseCntrl_{}.log'
.format(timezone.make_naive(value=timezone.now(), timezone=timezone.get_current_timezone()).strftime('%d-%m-%y_%H-%M-%S'))))
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
s_handler.setFormatter(formatter)
f_handler.setFormatter(formatter)
logger.addHandler(s_handler)
logger.addHandler(f_handler)
logger.setLevel(cfg.LOG_LEVEL)
def clean():
for sample in samples:
path = utils.get_sample_build_path(sample[0])
if os.path.exists(path):
shutil.rmtree(path)
remove_python_cache(utils.get_build_utils_path())
remove_python_cache(utils.get_samples_utils_path())
remove_python_cache(utils.get_script_path())
for root, subdirs, files in os.walk(utils.get_root_path()):
for file in files:
if file.endswith(".log"):
os.remove(os.path.join(root, file))
def ged(g1, g2, algo):
# https://github.com/dan-zam/graph-matching-toolkit
gp = get_gmt_path()
src, tp = setup_temp_folder(gp)
meta1 = write_to_temp(g1, tp, algo, 'g1')
meta2 = write_to_temp(g2, tp, algo, 'g2')
if meta1 != meta2:
raise RuntimeError('Different meta data {} vs {}'.format(meta1, meta2))
setup_property_file(src, gp, meta1)
if not exec(
'cd {} && java -classpath {}/src/graph-matching-toolkit/bin algorithms.GraphMatching ./properties/properties_temp_{}.prop'
.format(gp, get_root_path(), get_ts()),
timeout=1000):
return -1
return get_result(gp, algo)
def fit(self, train_X, train_Y, val_X=None, val_Y=None, graph=None):
"""Fit the model to the data.
Parameters
----------
train_X : array_like, shape (n_samples, n_features)
Training data.
train_Y : array_like, shape (n_samples, n_classes)
Training labels.
val_X : array_like, shape (N, n_features) optional, (default = None).
Validation data.
val_Y : array_like, shape (N, n_classes) optional, (default = None).
Validation labels.
graph : tf.Graph, optional (default = None)
Tensorflow Graph object.
Returns
-------
"""
if len(train_Y.shape) != 1:
num_classes = train_Y.shape[1]
else:
raise Exception("Please convert the labels with one-hot encoding.")
g = graph if graph is not None else self.tf_graph
with g.as_default():
# Build model
self.build_model(train_X.shape[1], num_classes)
with tf.Session() as self.tf_session:
# Initialize tf stuff
summary_objs = tf_utils.init_tf_ops(self.tf_session)
self.tf_merged_summaries = summary_objs[0]
self.tf_summary_writer = summary_objs[1]
self.tf_saver = summary_objs[2]
# Train model
self._train_model(train_X, train_Y, val_X, val_Y)
# Save model
_weight = self.tf_session.run(self.encoding_w_)
for matrix in _weight:
np.savetxt(utils.get_root_path(False) + '/save/' + str(matrix.shape[0]) + 'to' + str(matrix.shape[1]) + '.txt', matrix)
self.tf_saver.save(self.tf_session, self.model_path)
def get_contacts_for_country(variables):
f = open(get_root_path() + '/data/contact_matrix.csv', 'r')
max_age = variables['max_age']
df = pd.read_csv(f, header=0)
df = df[df.country == variables['country']].drop(columns='country')
df['place_type'] = df['place_type'].map(
lambda x: x.replace('cnt_', '').replace('otherplace', 'other'))
s = '-%d' % max_age
df['participant_age'] = df['participant_age'].map(
lambda x: x.replace('+', s))
last_col = [x for x in df.columns if '+' in x]
assert len(last_col) == 1
df = df.rename(columns={last_col[0]: last_col[0].replace('+', s)})
return df
def get_population():
f = open(get_root_path() + '/data/005_11re_2018.csv',
'r',
encoding='iso8859-1')
f.readline()
f.readline()
df = pd.read_csv(f)
df = df[(df.Alue != 'KOKO MAA') & (df['Ikä'] != 'Yhteensä')]
df = df.rename(
columns={
'Miehet 2018 Väestö 31.12.': 'Male',
'Naiset 2018 Väestö 31.12.': 'Female',
'Alue': 'Area',
'Ikä': 'Age',
})
df['Age'] = df['Age'].replace('100 -', '100').astype(int)
return df.set_index('Area')
def exp10():
# Query visualization.
dataset = 'aids10k'
model = 'beam80'
k = 5
info_dict = {
'draw_node_size': 10,
'draw_node_label_enable': True,
'draw_node_label_font_size': 8,
'draw_node_color_map': {
'C': 'red',
'O': 'blue',
'N': 'green'
},
'draw_edge_label_enable': True,
'draw_edge_label_font_size': 6,
'each_graph_text_list': [],
'each_graph_font_size': 10,
'plot_dpi': 200,
'plot_save_path': ''
}
r = load_result(dataset, model)
ged_mat = r.ged_mat()
time_mat = r.time_mat()
ids = r.ged_sort_id_mat()
m, n = ged_mat.shape
train_data = load_data(dataset, train=True)
test_data = load_data(dataset, train=False)
for i in range(m):
q = test_data.graphs[i]
gids = ids[i][:k]
gs = [train_data.graphs[j] for j in gids]
info_dict['each_graph_text_list'] = \
['query id: {}'.format(q.graph['gid'])] + \
[get_text_label(ged_mat, time_mat, i, j, \
train_data.graphs[j]) for j in gids]
info_dict['plot_save_path'] = \
get_root_path() + \
'/files/{}//query_vis/{}/query_vis_{}_{}_{}.png'.format( \
dataset, model, dataset, model, i)
vis(q, gs, info_dict)
def exp9():
# Plot ap@k.
dataset = 'aids10k'
models = ['beam5', 'beam10', 'beam20', 'beam40', 'beam80', \
'hungarian', 'vj']
true_model = 'beam80'
metric = 'ap@k'
rs = load_results_as_dict(dataset, models)
true_result = rs[true_model]
ks = []
k = 1
while k < true_result.ged_mat().shape[1]:
ks.append(k)
k *= 2
# print_ids = range(true_mat.shape[0])
print_ids = []
font = {'family': 'serif', 'size': 22}
matplotlib.rc('font', **font)
plt.figure(figsize=(16, 10))
for model in models:
print(model)
aps = precision_at_ks(true_result, rs[model], ks, print_ids)
#print('aps {}: {}'.format(model, aps))
plt.semilogx(ks, aps, **args1[model])
plt.scatter(ks, aps, s=200, label=model, **args2[model])
plt.xlabel('k')
# ax = plt.gca()
# ax.set_xticks(ks)
plt.ylabel(metric)
plt.legend(loc='best', ncol=2)
plt.grid(linestyle='dashed')
plt.tight_layout()
# plt.show()
plt.savefig(get_root_path() + '/files/{}/{}/ged_{}_{}_{}.png'.format( \
dataset, metric, metric, dataset, '_'.join(models)))
def exp4():
ms = [
'astar', 'beam5', 'beam10', 'beam20', 'beam40', 'beam80', 'hungarian',
'vj'
]
fn = '_'.join(ms)
file = open(get_root_path() + '/files/ged_{}_{}.csv'.format(fn, get_ts()),
'w')
xs = [10]
ys = list(range(10, 141, 10))
cnt = 10
ged_s = ','.join(['ged_' + i for i in ms])
time_s = ','.join(['time_' + i for i in ms])
print_and_log('g1_node,g2_node,g1_edge,g2_edge,{},{}'.format( \
ged_s, time_s), file)
for x in xs:
for y in ys:
for i in range(cnt):
g1 = generate_random_graph(x)
g2 = generate_random_graph(y)
ds = []
ts = []
for m in ms:
t = time()
d = ged(g1, g2, m)
t = time() - t
ds.append(d)
ts.append(t)
s = '{},{},{},{},{},{}'.format( \
g1.number_of_nodes(), g2.number_of_nodes(), \
g1.number_of_edges(), g2.number_of_edges(), \
','.join(str(i) for i in ds),
','.join(['{:.5f}'.format(i) for i in ts]))
print_and_log(s, file)
# if d1 < 0:
# exit(-1)
file.close()
def get_old_aids_id():
files = glob(get_root_path() + '/data/AIDS_old/data/*.gxl')
return [get_file_base_id(file) for file in files]
def gen_aids():
dirin = get_root_path() + '/data'
file = dirin + '/AIDO99SD'
dirout = dirin + '/AIDS'
g = nx.Graph()
line_i = 0
gid = 0
aids_old_ids = get_old_aids_id()
charges = set()
charge_gids = set()
with open(file) as f:
for line in f:
if '$$$$' in line:
print(gid)
nx.write_gexf(g, dirout + '/{}.gexf'.format(gid))
g = nx.Graph(gid=gid)
line_i = 0
gid += 1
else:
if gid == 40650:
print(line, end='')
ls = line.rstrip().split()
if len(ls) == 9:
nid = line_i - 4
ls = line.rstrip().split()
type = ls[3]
charge = int(ls[6])
# if charge != 0:
# print(gid)
# if (gid + 1) in aids_old_ids:
# print('#####'*20)
if charge != 0:
charges.add((type, charge))
charge_gids.add(gid)
if charge == 1:
charge = 3
elif charge == 2:
charge = 2
elif charge == 3:
charge = 1
elif charge == 4:
raise RuntimeError('Cannot handle doublet radical')
elif charge == 5:
charge = -1
elif charge == 6:
charge = -2
elif charge == 7:
charge = -3
elif charge != 0:
raise RuntimeError(
'Unrecognized charge {}'.format(charge))
if type != 'H':
g.add_node(nid, type=type)
elif len(ls) == 6:
ls = line.rstrip().split()
nid0 = int(ls[0]) - 1
nid1 = int(ls[1]) - 1
valence = int(ls[2])
if nid0 in g.nodes() and nid1 in g.nodes():
g.add_edge(nid0, nid1, valence=valence)
line_i += 1
print(len(charges), charges)
print(len(charge_gids), charge_gids)
# graph padding: value range: [0, 1]
'top_space':
0.1, # out of whole graph
'bottom_space':
0,
'hbetween_space':
0.4, # out of the subgraph
'wbetween_space':
0.01,
# plot config
'plot_dpi':
200,
'plot_save_path':
get_root_path() + '/temp/test_vis.png'
}
test_data = load_data('aids10k', train=False)
train_data = load_data('aids10k', train=True)
q = test_data.graphs[0]
gs = []
for i in range(1, 5):
print(i)
gs.append(train_data.graphs[i])
vis(q, gs, info_dict)
'''
TODO:
1. node color [done]
"""Library-wise configurations."""
import errno
import utils
import os
root = utils.get_root_path()
class Config(object):
"""Configuration class."""
class __Singleton(object):
"""Singleton design pattern."""
def __init__(self,
models_dir='models/',
data_dir='data/',
logs_dir='log/'):
"""Constructor.
Parameters
----------
models_dir : string, optional (default='models/')
directory path to store trained models.
Path is relative to ~/.yadlt
data_dir : string, optional (default='data/')
directory path to store model generated data.
Path is relative to ~/.yadlt
logs_dir : string, optional (default='logs/')
directory path to store yadlt and tensorflow logs.
Path is relative to ~/.yadlt
"""
def save(self):
self.folder, root_path = get_root_path(self.folder_name)
name = self.cleaned_data["file"].name
self.filename, self.path = get_file_path(name, root_path)
handle_uploaded_file(self.cleaned_data["file"], self.path)
for edge in edges:
g.add_edge(*edge)
return g
g0 = create_graph([(0, 1), (0, 2), (0, 3), (0, 4), (4, 5), (5, 6), \
(6, 7), (7, 5)])
g1 = create_graph([(0, 1), (2, 3)])
nx.set_node_attributes(g0, 'atom_type', {0: 'carbon', 1: 'oxygen'})
nx.set_node_attributes(g1, 'atom_type', {0: 'oxygen', 3: 'nitrogen'})
nx.set_node_attributes(g0, 'strawberry', {0: -0.01, 1: 3.61423, 2: 1423})
nx.set_edge_attributes(g0, 'some_label_1', {(0, 1): -1, (0, 2): -5, (7, \
5): 2.0})
nx.set_edge_attributes(g0, 'some_label_2', {(0, 1): 1, (0, 2): 5, (7, 5): 2})
nx.set_edge_attributes(g0, 'some_label_3', {(0, 1): True, (0, 2): False})
nx.set_edge_attributes(g0, 'some_label_4', {(0, 1): True, (0, 2): -0.9, \
(4, 5): 'xxx', (7, 5): '', \
(5, 6): None})
nx.set_edge_attributes(g0, 'some_label_5', {(0, 1): True, (0, 2): None})
nx.set_edge_attributes(g0, 'some_label_6', {(0, 1): True, (0, 2): [False]})
nx_to_gxl(g0, 'g0', get_root_path() + '/files/g0.gxl')
nx_to_gxl(g1, 'g1', get_root_path() + '/files/g1.gxl')
def gen_aids_small(name, additional=False):
datadir = get_root_path() + '/data'
dirin = datadir + '/AIDS40k_orig'
sfn = get_save_path() + '/aids40k_orig'
loaded = load_as_dict(sfn)
if not loaded:
graphs = {}
nodes_graphs = defaultdict(list)
lesseq30 = set()
lesseq10 = set()
disconnects = set()
# Iterate through all 40k graphs.
for file in glob(dirin + '/*.gexf'):
gid = int(file.split('/')[-1].split('.')[0])
g = nx.read_gexf(file)
if not nx.is_connected(g):
print('{} not connected'.format(gid))
disconnects.add(gid)
continue
graphs[gid] = g
nodes_graphs[g.number_of_nodes()].append(gid)
if g.number_of_nodes() <= 30:
lesseq30.add(gid)
if g.number_of_nodes() <= 10:
lesseq10.add(gid)
save_as_dict(sfn, graphs, nodes_graphs, lesseq30, lesseq10,
disconnects)
else:
graphs = loaded['graphs']
nodes_graphs = loaded['nodes_graphs']
lesseq30 = loaded['lesseq30']
lesseq10 = loaded['lesseq10']
disconnects = loaded['disconnects']
print(len(disconnects), 'disconnected graphs out of', len(graphs))
print(len(lesseq30), 'with <= 30 nodes')
print(len(lesseq10), 'with <= 10 nodes')
# exit(1)
train_dir = '{}/{}/train'.format(datadir, name)
if additional:
train_data = load_data(name.lower(), train=True)
test_dir_str = 'test2'
else:
exec_cmd('mkdir -p {}'.format(train_dir))
test_dir_str = 'test'
test_dir = '{}/{}/{}'.format(datadir, name, test_dir_str)
exec_cmd('mkdir -p {}'.format(test_dir))
if not additional:
if name == 'AIDS10k':
for num_node in range(5, 23):
choose = random.Random(123).sample(nodes_graphs[num_node],
1)[0]
print('choose {} with {} nodes'.format(choose, num_node))
nx.write_gexf(graphs[choose],
test_dir + '/{}.gexf'.format(choose))
lesseq30.remove(choose)
for tid in random.Random(123).sample(lesseq30, 10000):
nx.write_gexf(graphs[tid], train_dir + '/{}.gexf'.format(tid))
elif name == 'AIDS700nef':
lesseq10 = sample_from_lessthan10eq(train_dir, lesseq10, 560,
graphs, 'train')
sample_from_lessthan10eq(test_dir, lesseq10, 140, graphs, 'test')
else:
assert (name == 'AIDS10k')
for num_node in range(5, 30):
k = 4
from_li = nodes_graphs[num_node]
print('sampling {} from {} (size={})'.format(
k, len(from_li), num_node))
choose = random.Random(123).sample_exclude(from_li, k,
train_data.get_gids())
print('choose {} with {} nodes'.format(choose, num_node))
for c in choose:
nx.write_gexf(graphs[c], test_dir + '/{}.gexf'.format(c))
print('Done')
def __init__(self, model):
super(BaseGenerator, self).__init__()
self.model = model
self.templates_path = os.path.join(get_root_path(), "genodis",
TEMPLATES_PATH)
self.templates_folder_name = ""
def get_healthcare_districts():
p = get_root_path() + '/data/shp_jasenkunnat_2020.xls'
df = pd.read_excel(p, header=3, sheet_name='shp_jäsenkunnat_2020_lkm')
df = df[['kunta', 'sairaanhoitopiiri', 'erva-alue']].dropna()
return df
def generate(self):
root_path = get_root_path()
jinja_env = self.setup_env()
d = {"model": self.model, "app_name": self.model.name.lower()}
destination = os.path.join(root_path, SRC_GEN_PATH)
if not os.path.exists(destination):
os.mkdir(destination)
django_path = os.path.join(destination, self.model.name.lower())
os.mkdir(django_path)
#
# Create manage.py file
#
template = jinja_env.get_template("django_manage.template")
template.stream(d).dump(os.path.join(django_path, 'manage.py'))
#
# Create folder with settings.py, urls.py and wsgi.py
#
app_settings = os.path.join(django_path, model.name.lower())
os.mkdir(app_settings)
self.create_init_file(app_settings)
template = jinja_env.get_template("django_settings.template")
template.stream(d).dump(os.path.join(app_settings, 'settings.py'))
template = jinja_env.get_template("django_urls.template")
template.stream(d).dump(os.path.join(app_settings, 'urls.py'))
template = jinja_env.get_template("django_wsgi.template")
template.stream(d).dump(os.path.join(app_settings, 'wsgi.py'))
#
# Create app folder
#
django_app = os.path.join(django_path, model.name.lower() + "_app")
os.mkdir(django_app)
self.create_init_file(django_app)
#
# Creating admin.py file
#
template = jinja_env.get_template("django_admin.template")
template.stream(d).dump(os.path.join(django_app, 'admin.py'))
#
# Creating models.py file
#
template = jinja_env.get_template("django_models.template")
template.stream(d).dump(os.path.join(django_app, 'models.py'))
#
# Creating tests.py file
#
template = jinja_env.get_template("django_tests.template")
template.stream(d).dump(os.path.join(django_app, 'tests.py'))
#
# Creating urls.py file
#
template = jinja_env.get_template("django_urls.template")
template.stream(d).dump(os.path.join(django_app, 'urls.py'))
#
# Creating views.py file
#
template = jinja_env.get_template("django_views.template")
template.stream(d).dump(os.path.join(django_app, 'views.py'))
# creating folder for migrations
migrations_path = os.path.join(django_app, "migrations")
os.mkdir(migrations_path)
self.create_init_file(migrations_path)
# creating folder for templates
templates_path = os.path.join(django_app, "templates")
os.mkdir(templates_path)
view_templates = os.path.join(templates_path,
self.model.name.lower())
os.mkdir(view_templates)
# creating folder for static files
static_files = os.path.join(django_app, "static")
os.mkdir(static_files)
def generate(self):
root_path = get_root_path()
jinja_env = self.setup_env()
d = {"model": self.model, "app_name": self.model.name.lower()}
destination = os.path.join(root_path, SRC_GEN_PATH)
if not self.generate_as_static:
if not os.path.exists(destination):
os.mkdir(destination)
angular_app_name = self.model.name.lower() + "_js"
angular_path = os.path.join(destination, angular_app_name)
os.mkdir(angular_path)
# create app folder
angular_app = os.path.join(angular_path, "app")
os.mkdir(angular_app)
else:
angular_app = os.path.join(destination, model.name.lower(),
model.name.lower() + "_app", "static")
#
# Create app.js
#
template = jinja_env.get_template("app.template")
template.stream(d).dump(os.path.join(angular_app, 'app.js'))
#
# Create app.css
#
template = jinja_env.get_template("app_css.template")
template.stream(d).dump(os.path.join(angular_app, 'app.css'))
#
# Create index.html
#
template = jinja_env.get_template("index.template")
template.stream(d).dump(os.path.join(angular_app, 'index.html'))
# create controllers folder
controllers_path = os.path.join(angular_app, "controllers")
os.mkdir(controllers_path)
#
# Create controllers for all classes
#
template = jinja_env.get_template("controllers.template")
for c in self.model.classes:
ctlr_name = c.name.lower() + "_controller.js"
data = {"c": c}
template.stream(data).dump(os.path.join(controllers_path,
ctlr_name))
# create views folder
views_path = os.path.join(angular_app, "views")
os.mkdir(views_path)
#
# Create views for all classes
#
template = jinja_env.get_template("views.template")
for c in self.model.classes:
view_name = c.name.lower() + "s.html"
data = {"c": c}
template.stream(data).dump(os.path.join(views_path, view_name))
def get_js_ctx(file_name):
js_dir = os.path.join(get_root_path(__name__), 'encrypt_file', file_name)
jsstr = file_reader(js_dir)
ctx = execjs.compile(jsstr)
return ctx
ctlr_name = c.name.lower() + "_controller.js"
data = {"c": c}
template.stream(data).dump(os.path.join(controllers_path,
ctlr_name))
# create views folder
views_path = os.path.join(angular_app, "views")
os.mkdir(views_path)
#
# Create views for all classes
#
template = jinja_env.get_template("views.template")
for c in self.model.classes:
view_name = c.name.lower() + "s.html"
data = {"c": c}
template.stream(data).dump(os.path.join(views_path, view_name))
if __name__ == "__main__":
metamodel = get_model_meta()
path = get_root_path()
model_file = os.path.join(path, "tests", "examples", "simple_model.gm")
model = metamodel.model_from_file(model_file)
server_generator = DjangoServerGenerator(model)
server_generator.generate()
angular_generator = AngularJSGenerator(model, generate_as_static=True)
angular_generator.generate()