def generate_postfix_slm_map(giedo): # We generate the postfix "sender_login_maps". # This is used to decide by postfix whether a given user is allowed to # send e-mail as if it was coming from a particular e-mail address. # It is a dictionary { <email address> : [ <allowed user>, ... ] } tbl = dict() dt_now = now() # Get all users ulut = dict() # We only allow members to send e-mail for u in Es.by_name('leden').get_members(): ulut[u._id] = u for name in u.names: if str(name) not in tbl: tbl[str(name)] = set() tbl[str(name)].add(str(u.name)) # There are two kind of groups: groups whose members are allowed to # send e-mail as if coming from the group itself and those where this # is not allowed. For convenience, lets call the first kind the # impersonatable groups. # Get all impersonatable groups and create a look-up-table for # group membership gs = list() for g in Es.groups(): # TODO add a tag to force a group either impersonatable or not if not g.got_mailman_list: gs.append(g) mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now) mlut = dict() for g in gs: mlut[g._id] = [] for mrel in mrels: mlut[mrel['with']].append(mrel['who']) # Flatten out group membership. For instance: if Giedo is in Kasco # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers # unix group. # But first split the mlut graph into a impersonatable group # and a non-group subgraph. mlut_g = {} # { <group> : <members that are impersonatable groups> } mlut_u = {} # { <group> : <members that are users> } for g_id in mlut: mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut] mlut_u[g_id] = [c for c in mlut[g_id] if c in ulut] mlut_g_tc = tc(mlut_g) # transitive closure for g in gs: to_consider = tuple(mlut_g_tc[g._id]) + (g._id,) for sg_id in to_consider: for u_id in mlut_u[sg_id]: for name in g.names: if str(name) not in tbl: tbl[str(name)] = set() tbl[str(name)].add(str(ulut[u_id].name)) # Clean up tbl to return. ret = {} for name, users in six.iteritems(tbl): if not users: continue ret["%s@%s" % (name, settings.MAILDOMAIN)] = tuple(users) return ret
def generate_unix_map(giedo): ret = {'groups': {}, 'users': {}} dt_now = now() # Get all users ulut = dict() for u in Es.users(): if not u.got_unix_user: continue ulut[u._id] = u ret['users'][str(u.name)] = { 'full_name': u.full_name, 'expire_date': DT_MIN.strftime('%Y-%m-%d')} member_relations_grouped = dict() for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now): if rel['who'] not in member_relations_grouped: member_relations_grouped[rel['who']] = [] member_relations_grouped[rel['who']].append(rel) for user_id, relations in member_relations_grouped.items(): latest = max(relations, key=lambda x: x['until']) ret['users'][str(ulut[user_id].name)]['expire_date'] \ = latest['until'].strftime('%Y-%m-%d') # Get all groups and create a look-up-table for group membership gs = tuple(Es.groups()) mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now) mlut = dict() for g in gs: mlut[g._id] = [] for mrel in mrels: mlut[mrel['with']].append(mrel['who']) # Flatten out group membership. For instance: if Giedo is in Kasco # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers # unix group. # But first split the mlut graph into a group and a non-group subgraph. mlut_g = {} # { <group> : <members that are groups> } mlut_ng = {} # { <group> : <members that are not groups> } for g_id in mlut: mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut] mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut] mlut_g_tc = tc(mlut_g) # transitive closure # Generate the { <group> : <indirect non-group members> } graph memb_graph = {} for g in gs: if not g.got_unix_group: continue memb_graph[g._id] = set(mlut_ng[g._id]) for h_id in mlut_g_tc[g._id]: memb_graph[g._id].update(mlut_ng[h_id]) # Fill the return map for g in gs: if not g.got_unix_group: continue ret['groups'][str(g.name)] = [str(ulut[c].name) for c in memb_graph[g._id] if c in ulut] return ret
def generate_unix_map(giedo): ret = {'groups': {}, 'users': {}} dt_now = now() # Get all users ulut = dict() for u in Es.users(): if not u.got_unix_user: continue ulut[u._id] = u ret['users'][str(u.name)] = { 'full_name': u.full_name, 'expire_date': DT_MIN.strftime('%Y-%m-%d')} member_relations_grouped = dict() for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now): if rel['who'] not in member_relations_grouped: member_relations_grouped[rel['who']] = [] member_relations_grouped[rel['who']].append(rel) for user_id, relations in member_relations_grouped.items(): latest = max(relations, key=lambda x: x['until']) ret['users'][str(ulut[user_id].name)]['expire_date'] = ( latest['until'].strftime('%Y-%m-%d')) # Get all groups and create a look-up-table for group membership gs = tuple(Es.groups()) mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now) mlut = dict() for g in gs: mlut[g._id] = [] for mrel in mrels: mlut[mrel['with']].append(mrel['who']) # Flatten out group membership. For instance: if Giedo is in Kasco # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers # unix group. # But first split the mlut graph into a group and a non-group subgraph. mlut_g = {} # { <group> : <members that are groups> } mlut_ng = {} # { <group> : <members that are not groups> } for g_id in mlut: mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut] mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut] mlut_g_tc = tc(mlut_g) # transitive closure # Generate the { <group> : <indirect non-group members> } graph memb_graph = {} for g in gs: if not g.got_unix_group: continue memb_graph[g._id] = set(mlut_ng[g._id]) for h_id in mlut_g_tc[g._id]: memb_graph[g._id].update(mlut_ng[h_id]) # Fill the return map for g in gs: if not g.got_unix_group: continue ret['groups'][str(g.name)] = [str(ulut[c].name) for c in memb_graph[g._id] if c in ulut] return ret
def find_sets(graph): edges = [pair for (pair, w) in graph.edgelist] vertices = graph.vertices.keys() if not edges: return [random.choice(vertices)] paths = OD() t_input = {v: graph[v].edges.keys() for v in graph.vertices} # print(t_input) scc_index = random.choice(vertices) max_scc = 0 ''' closure = tarjan(t_input) for idx,lv in enumerate(closure): if len(lv) > max_scc: max_scc = len(lv) scc_index = idx print(idx, len(lv), lv) ''' closure = tc(t_input) for idx, path in closure.iteritems(): if len(path) > max_scc: max_scc = len(path) scc_index = idx # print(idx, len(path), path) def calc_score(vertices): return len(vertices) * sum(graph[v].value for v in vertices) print(scc_index, max_scc) best_path = [scc_index] best_score = calc_score(best_path) print(best_score, best_path) verify_path(graph, best_path) paths = [] ctr = 0 for i in range(1000): path = dfs_extend(graph, best_path, closure) paths.append((calc_score(path), path)) if len(paths) < 10: ctr += 1 if i < 500 and ctr > 200: break elif i < 100 and ctr > 50: break elif i < 50 and ctr > 25: break elif i < 10 and ctr > 6: break sorted_paths = sorted(paths, reverse=True) print(sorted_paths[0]) best_path = sorted_paths[0][1] return best_path
def calConnectivityRatio(A): G = dict() for i in range(len(A)): G[i] = [] for j in range(len(A[i])): if i == j: G[i].append(j) elif A[i][j] == 1: G[i].append(j) groups = tc(G) groups:dict difGroups = set() for value in groups.values(): difGroups.add(value) if len(difGroups) != 0: return (1./len(difGroups))**(1/3) else: return 0.
def epsionTransitionDestList(transitions): singular = {} for i in range(len(transitions)): t = transitions[i] singular[i] = t[2] epsilonTransitions = tc(singular) result = [] for i in range(len(epsilonTransitions)): current = epsilonTransitions[i] bitStr = '' for j in range(len(epsilonTransitions)): if j in current: bitStr = '1' + bitStr else: bitStr = '0' + bitStr result.append(bitStr) return result
def generate_wolk_changes(giedo): creds = settings.WOLK_MYSQL_SECRET if not creds: logging.warning('wolk: no credentials available, skipping') return None todo = {'addUser': [], 'addGroup': [], 'addUserToGroup': [], 'removeUserFromGroup': []} dt_now = now() # First, lets see which users and groups to create users = dict() # users to have groups = dict() # groups to have missing_groups = set() missing_users = set() ulut = dict() for m in Es.by_name('leden').get_members(): if not m.got_unix_user: continue ulut[m._id] = m users[str(m.name)] = m # Get all groups and create a look-up-table for group membership gs = tuple(Es.groups()) mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now) mlut = dict() for g in gs: mlut[g._id] = [] for mrel in mrels: mlut[mrel['with']].append(mrel['who']) # Flatten out group membership. For instance: if Giedo is in Kasco # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers # unix group. # But first split the mlut graph into a group and a non-group subgraph. mlut_g = {} # { <group> : <members that are groups> } mlut_ng = {} # { <group> : <members that are not groups> } for g_id in mlut: mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut] mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut] mlut_g_tc = tc(mlut_g) # transitive closure # Generate the { <group> : <indirect non-group members> } graph memb_graph = {} for g in gs: if not g.got_unix_group: continue memb_graph[g._id] = set(mlut_ng[g._id]) for h_id in mlut_g_tc[g._id]: memb_graph[g._id].update(mlut_ng[h_id]) # Fill the groups variable for g in gs: if not g.got_unix_group: continue groups[str(g.name)] = set([str(ulut[c].name) for c in memb_graph[g._id] if c in ulut]) # Now, check which users and groups actually exist in owncloud missing_users = set(users.keys()) missing_groups = set(groups.keys()) dc = pymysql.connect( host=creds[0], user=creds[1], password=creds[2], db=creds[3], charset='utf8' ) try: with dc.cursor() as c: c.execute("SELECT gid, uid FROM oc_group_user") for group, user in c.fetchall(): if group not in groups: continue if user not in users or user not in groups[group]: todo['removeUserFromGroup'].append((user, group)) continue if user in groups[group]: groups[group].remove(user) c.execute("SELECT uid FROM oc_users") for user, in c.fetchall(): if user not in users: logging.info("wolk: stray user %s", user) continue missing_users.remove(user) c.execute("SELECT gid FROM oc_groups") for group, in c.fetchall(): if group not in groups: logging.info("wolk: stray group %s", user) continue missing_groups.remove(group) for user in missing_users: todo['addUser'].append(( user, six.text_type(users[user].humanName) )) todo['addGroup'] = list(missing_groups) for group, missing_members in six.iteritems(groups): for user in missing_members: todo['addUserToGroup'].append((user, group)) finally: dc.close() return todo
def find_sets(graph): edges = [pair for (pair, w) in graph.edgelist] vertices = graph.vertices.keys() if not edges: return [random.choice(vertices)] def calc_score(vertices): return len(vertices) * sum(graph[v].value for v in vertices) visited = OD() t_input = {v: graph[v].edges.keys() for v in graph.vertices} scc_index = random.choice(vertices) max_scc = 0 closure = tc(t_input) for idx, path in closure.iteritems(): if len(path) > max_scc: max_scc = len(path) scc_index = idx best_path = [scc_index] ''' h = [ (1,v) for v in vertices ] random.shuffle(h) ''' h = [(1, scc_index)] levels = OD() levels[1] = OD(zip([t[1] for t in h], ['NULL'] * len(h))) heapify(h) best_path = [random.choice(vertices)] best_score = calc_score(best_path) i = 0 while h: print("Iteration %s" % i) i += 1 lv, node = heappop(h) nxt = abs(lv) + 1 if node in visited: continue visited[node] = True children = graph[node].edges.keys() random.shuffle(children) for child in children: if not levels.get(nxt): levels[nxt] = OD([(child, node)]) else: levels[nxt][child] = node if child not in visited: heappush(h, (-nxt, child)) for (k, v) in levels.iteritems(): print(k, v) depth = max(levels.keys()) start = random.choice(levels[depth].keys()) prev = levels[depth][start] path = OD([(start, True)]) print("Backtracking") for j in range(depth - 1, 0, -1): print(j, prev, path.keys()) if prev != 'NULL': if prev in path: print("ERROR: Prev in path") if prev == path.keys()[0]: path.popitem(last=False) # print("Adding %s to %s" % (prev, path.keys())) path[prev] = True else: break prev = levels[j][prev] best_path = [k for k in reversed(path.keys())] best_score = calc_score(best_path) print(best_score, best_path) verify_path(graph, best_path) if 0.99 > random.random(): best_path = dfs_extend(graph, best_path, closure) if 0.99 > random.random(): best_path = prefix_extend(graph, best_path) return best_path
def generate_wolk_changes(giedo): creds = settings.WOLK_MYSQL_SECRET if not creds: logging.warning('wolk: no credentials available, skipping') return None todo = { 'addUser': [], 'addGroup': [], 'addUserToGroup': [], 'removeUserFromGroup': [] } dt_now = now() # First, lets see which users and groups to create users = dict() # users to have groups = dict() # groups to have missing_groups = set() missing_users = set() ulut = dict() for m in Es.by_name('leden').get_members(): if not m.got_unix_user: continue ulut[m._id] = m users[str(m.name)] = m # Get all groups and create a look-up-table for group membership gs = tuple(Es.groups()) mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now) mlut = dict() for g in gs: mlut[g._id] = [] for mrel in mrels: mlut[mrel['with']].append(mrel['who']) # Flatten out group membership. For instance: if Giedo is in Kasco # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers # unix group. # But first split the mlut graph into a group and a non-group subgraph. mlut_g = {} # { <group> : <members that are groups> } mlut_ng = {} # { <group> : <members that are not groups> } for g_id in mlut: mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut] mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut] mlut_g_tc = tc(mlut_g) # transitive closure # Generate the { <group> : <indirect non-group members> } graph memb_graph = {} for g in gs: if not g.got_unix_group: continue memb_graph[g._id] = set(mlut_ng[g._id]) for h_id in mlut_g_tc[g._id]: memb_graph[g._id].update(mlut_ng[h_id]) # Fill the groups variable for g in gs: if not g.got_unix_group: continue groups[str(g.name)] = set( [str(ulut[c].name) for c in memb_graph[g._id] if c in ulut]) # Now, check which users and groups actually exist in owncloud missing_users = set(users.keys()) missing_groups = set(groups.keys()) dc = pymysql.connect(host=creds[0], user=creds[1], password=creds[2], db=creds[3], charset='utf8') try: with dc.cursor() as c: c.execute("SELECT gid, uid FROM oc_group_user") for group, user in c.fetchall(): if group not in groups: continue if user not in users or user not in groups[group]: todo['removeUserFromGroup'].append((user, group)) continue if user in groups[group]: groups[group].remove(user) c.execute("SELECT uid FROM oc_users") for user, in c.fetchall(): if user not in users: logging.info("wolk: stray user %s", user) continue missing_users.remove(user) c.execute("SELECT gid FROM oc_groups") for group, in c.fetchall(): if group not in groups: logging.info("wolk: stray group %s", user) continue missing_groups.remove(group) for user in missing_users: todo['addUser'].append( (user, six.text_type(users[user].humanName))) todo['addGroup'] = list(missing_groups) for group, missing_members in six.iteritems(groups): for user in missing_members: todo['addUserToGroup'].append((user, group)) finally: dc.close() return todo
sys.path = [ '/usr/lib/python2.7', '/usr/local/lib/python2.7/dist-packages/tarjan-0.2.3.2-py2.7.egg' ] from tarjan.tc import tc input_path = str(sys.argv[1]) print "~TARJAN CHECK~" print input_path with open(input_path, "r") as json_file: graph = json.load(json_file) num_nodes = len(graph) scc = tc(graph) print len(scc.keys()) for k, v in scc.iteritems(): if len(v) != num_nodes: excluded = [] for node in scc.keys(): if not node in v: excluded.append(node) print "============================================" print "ERROR: the whole graph is not a single SCC!" print "The number of expected nodes = " + str(num_nodes) print "Differs from the number of actual nodes = " + str(len(v)) print "The ISOLATED SCC is the following : " print v print "The key associated with this error is the following:" print k