def generate_postfix_slm_map(giedo):
# We generate the postfix "sender_login_maps".
# This is used to decide by postfix whether a given user is allowed to
# send e-mail as if it was coming from a particular e-mail address.
# It is a dictionary { <email address> : [ <allowed user>, ... ] }
tbl = dict()
dt_now = now()
# Get all users
ulut = dict()
# We only allow members to send e-mail
for u in Es.by_name('leden').get_members():
ulut[u._id] = u
for name in u.names:
if str(name) not in tbl:
tbl[str(name)] = set()
tbl[str(name)].add(str(u.name))
# There are two kind of groups: groups whose members are allowed to
# send e-mail as if coming from the group itself and those where this
# is not allowed. For convenience, lets call the first kind the
# impersonatable groups.
# Get all impersonatable groups and create a look-up-table for
# group membership
gs = list()
for g in Es.groups():
# TODO add a tag to force a group either impersonatable or not
if not g.got_mailman_list:
gs.append(g)
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a impersonatable group
# and a non-group subgraph.
mlut_g = {} # { <group> : <members that are impersonatable groups> }
mlut_u = {} # { <group> : <members that are users> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_u[g_id] = [c for c in mlut[g_id] if c in ulut]
mlut_g_tc = tc(mlut_g) # transitive closure
for g in gs:
to_consider = tuple(mlut_g_tc[g._id]) + (g._id,)
for sg_id in to_consider:
for u_id in mlut_u[sg_id]:
for name in g.names:
if str(name) not in tbl:
tbl[str(name)] = set()
tbl[str(name)].add(str(ulut[u_id].name))
# Clean up tbl to return.
ret = {}
for name, users in six.iteritems(tbl):
if not users:
continue
ret["%s@%s" % (name, settings.MAILDOMAIN)] = tuple(users)
return ret
def generate_unix_map(giedo):
ret = {'groups': {},
'users': {}}
dt_now = now()
# Get all users
ulut = dict()
for u in Es.users():
if not u.got_unix_user:
continue
ulut[u._id] = u
ret['users'][str(u.name)] = {
'full_name': u.full_name,
'expire_date': DT_MIN.strftime('%Y-%m-%d')}
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
ret['users'][str(ulut[user_id].name)]['expire_date'] \
= latest['until'].strftime('%Y-%m-%d')
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the return map
for g in gs:
if not g.got_unix_group:
continue
ret['groups'][str(g.name)] = [str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut]
return ret
def generate_unix_map(giedo):
ret = {'groups': {},
'users': {}}
dt_now = now()
# Get all users
ulut = dict()
for u in Es.users():
if not u.got_unix_user:
continue
ulut[u._id] = u
ret['users'][str(u.name)] = {
'full_name': u.full_name,
'expire_date': DT_MIN.strftime('%Y-%m-%d')}
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
ret['users'][str(ulut[user_id].name)]['expire_date'] = (
latest['until'].strftime('%Y-%m-%d'))
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the return map
for g in gs:
if not g.got_unix_group:
continue
ret['groups'][str(g.name)] = [str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut]
return ret
def find_sets(graph):
edges = [pair for (pair, w) in graph.edgelist]
vertices = graph.vertices.keys()
if not edges:
return [random.choice(vertices)]
paths = OD()
t_input = {v: graph[v].edges.keys() for v in graph.vertices}
# print(t_input)
scc_index = random.choice(vertices)
max_scc = 0
'''
closure = tarjan(t_input)
for idx,lv in enumerate(closure):
if len(lv) > max_scc:
max_scc = len(lv)
scc_index = idx
print(idx, len(lv), lv)
'''
closure = tc(t_input)
for idx, path in closure.iteritems():
if len(path) > max_scc:
max_scc = len(path)
scc_index = idx
# print(idx, len(path), path)
def calc_score(vertices):
return len(vertices) * sum(graph[v].value for v in vertices)
print(scc_index, max_scc)
best_path = [scc_index]
best_score = calc_score(best_path)
print(best_score, best_path)
verify_path(graph, best_path)
paths = []
ctr = 0
for i in range(1000):
path = dfs_extend(graph, best_path, closure)
paths.append((calc_score(path), path))
if len(paths) < 10:
ctr += 1
if i < 500 and ctr > 200:
break
elif i < 100 and ctr > 50:
break
elif i < 50 and ctr > 25:
break
elif i < 10 and ctr > 6:
break
sorted_paths = sorted(paths, reverse=True)
print(sorted_paths[0])
best_path = sorted_paths[0][1]
return best_path
def calConnectivityRatio(A):
G = dict()
for i in range(len(A)):
G[i] = []
for j in range(len(A[i])):
if i == j:
G[i].append(j)
elif A[i][j] == 1:
G[i].append(j)
groups = tc(G)
groups:dict
difGroups = set()
for value in groups.values():
difGroups.add(value)
if len(difGroups) != 0:
return (1./len(difGroups))**(1/3)
else:
return 0.
def epsionTransitionDestList(transitions):
singular = {}
for i in range(len(transitions)):
t = transitions[i]
singular[i] = t[2]
epsilonTransitions = tc(singular)
result = []
for i in range(len(epsilonTransitions)):
current = epsilonTransitions[i]
bitStr = ''
for j in range(len(epsilonTransitions)):
if j in current:
bitStr = '1' + bitStr
else:
bitStr = '0' + bitStr
result.append(bitStr)
return result
def generate_wolk_changes(giedo):
creds = settings.WOLK_MYSQL_SECRET
if not creds:
logging.warning('wolk: no credentials available, skipping')
return None
todo = {'addUser': [], 'addGroup': [],
'addUserToGroup': [], 'removeUserFromGroup': []}
dt_now = now()
# First, lets see which users and groups to create
users = dict() # users to have
groups = dict() # groups to have
missing_groups = set()
missing_users = set()
ulut = dict()
for m in Es.by_name('leden').get_members():
if not m.got_unix_user:
continue
ulut[m._id] = m
users[str(m.name)] = m
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the groups variable
for g in gs:
if not g.got_unix_group:
continue
groups[str(g.name)] = set([str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut])
# Now, check which users and groups actually exist in owncloud
missing_users = set(users.keys())
missing_groups = set(groups.keys())
dc = pymysql.connect(
host=creds[0],
user=creds[1],
password=creds[2],
db=creds[3],
charset='utf8'
)
try:
with dc.cursor() as c:
c.execute("SELECT gid, uid FROM oc_group_user")
for group, user in c.fetchall():
if group not in groups:
continue
if user not in users or user not in groups[group]:
todo['removeUserFromGroup'].append((user, group))
continue
if user in groups[group]:
groups[group].remove(user)
c.execute("SELECT uid FROM oc_users")
for user, in c.fetchall():
if user not in users:
logging.info("wolk: stray user %s", user)
continue
missing_users.remove(user)
c.execute("SELECT gid FROM oc_groups")
for group, in c.fetchall():
if group not in groups:
logging.info("wolk: stray group %s", user)
continue
missing_groups.remove(group)
for user in missing_users:
todo['addUser'].append((
user,
six.text_type(users[user].humanName)
))
todo['addGroup'] = list(missing_groups)
for group, missing_members in six.iteritems(groups):
for user in missing_members:
todo['addUserToGroup'].append((user, group))
finally:
dc.close()
return todo
def find_sets(graph):
edges = [pair for (pair, w) in graph.edgelist]
vertices = graph.vertices.keys()
if not edges:
return [random.choice(vertices)]
def calc_score(vertices):
return len(vertices) * sum(graph[v].value for v in vertices)
visited = OD()
t_input = {v: graph[v].edges.keys() for v in graph.vertices}
scc_index = random.choice(vertices)
max_scc = 0
closure = tc(t_input)
for idx, path in closure.iteritems():
if len(path) > max_scc:
max_scc = len(path)
scc_index = idx
best_path = [scc_index]
'''
h = [ (1,v) for v in vertices ]
random.shuffle(h)
'''
h = [(1, scc_index)]
levels = OD()
levels[1] = OD(zip([t[1] for t in h], ['NULL'] * len(h)))
heapify(h)
best_path = [random.choice(vertices)]
best_score = calc_score(best_path)
i = 0
while h:
print("Iteration %s" % i)
i += 1
lv, node = heappop(h)
nxt = abs(lv) + 1
if node in visited:
continue
visited[node] = True
children = graph[node].edges.keys()
random.shuffle(children)
for child in children:
if not levels.get(nxt):
levels[nxt] = OD([(child, node)])
else:
levels[nxt][child] = node
if child not in visited:
heappush(h, (-nxt, child))
for (k, v) in levels.iteritems():
print(k, v)
depth = max(levels.keys())
start = random.choice(levels[depth].keys())
prev = levels[depth][start]
path = OD([(start, True)])
print("Backtracking")
for j in range(depth - 1, 0, -1):
print(j, prev, path.keys())
if prev != 'NULL':
if prev in path:
print("ERROR: Prev in path")
if prev == path.keys()[0]:
path.popitem(last=False)
# print("Adding %s to %s" % (prev, path.keys()))
path[prev] = True
else:
break
prev = levels[j][prev]
best_path = [k for k in reversed(path.keys())]
best_score = calc_score(best_path)
print(best_score, best_path)
verify_path(graph, best_path)
if 0.99 > random.random():
best_path = dfs_extend(graph, best_path, closure)
if 0.99 > random.random():
best_path = prefix_extend(graph, best_path)
return best_path
def generate_wolk_changes(giedo):
creds = settings.WOLK_MYSQL_SECRET
if not creds:
logging.warning('wolk: no credentials available, skipping')
return None
todo = {
'addUser': [],
'addGroup': [],
'addUserToGroup': [],
'removeUserFromGroup': []
}
dt_now = now()
# First, lets see which users and groups to create
users = dict() # users to have
groups = dict() # groups to have
missing_groups = set()
missing_users = set()
ulut = dict()
for m in Es.by_name('leden').get_members():
if not m.got_unix_user:
continue
ulut[m._id] = m
users[str(m.name)] = m
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the groups variable
for g in gs:
if not g.got_unix_group:
continue
groups[str(g.name)] = set(
[str(ulut[c].name) for c in memb_graph[g._id] if c in ulut])
# Now, check which users and groups actually exist in owncloud
missing_users = set(users.keys())
missing_groups = set(groups.keys())
dc = pymysql.connect(host=creds[0],
user=creds[1],
password=creds[2],
db=creds[3],
charset='utf8')
try:
with dc.cursor() as c:
c.execute("SELECT gid, uid FROM oc_group_user")
for group, user in c.fetchall():
if group not in groups:
continue
if user not in users or user not in groups[group]:
todo['removeUserFromGroup'].append((user, group))
continue
if user in groups[group]:
groups[group].remove(user)
c.execute("SELECT uid FROM oc_users")
for user, in c.fetchall():
if user not in users:
logging.info("wolk: stray user %s", user)
continue
missing_users.remove(user)
c.execute("SELECT gid FROM oc_groups")
for group, in c.fetchall():
if group not in groups:
logging.info("wolk: stray group %s", user)
continue
missing_groups.remove(group)
for user in missing_users:
todo['addUser'].append(
(user, six.text_type(users[user].humanName)))
todo['addGroup'] = list(missing_groups)
for group, missing_members in six.iteritems(groups):
for user in missing_members:
todo['addUserToGroup'].append((user, group))
finally:
dc.close()
return todo
sys.path = [
'/usr/lib/python2.7',
'/usr/local/lib/python2.7/dist-packages/tarjan-0.2.3.2-py2.7.egg'
]
from tarjan.tc import tc
input_path = str(sys.argv[1])
print "~TARJAN CHECK~"
print input_path
with open(input_path, "r") as json_file:
graph = json.load(json_file)
num_nodes = len(graph)
scc = tc(graph)
print len(scc.keys())
for k, v in scc.iteritems():
if len(v) != num_nodes:
excluded = []
for node in scc.keys():
if not node in v:
excluded.append(node)
print "============================================"
print "ERROR: the whole graph is not a single SCC!"
print "The number of expected nodes = " + str(num_nodes)
print "Differs from the number of actual nodes = " + str(len(v))
print "The ISOLATED SCC is the following : "
print v
print "The key associated with this error is the following:"
print k