def generate_unix_map(giedo):
ret = {'groups': {},
'users': {}}
dt_now = now()
# Get all users
ulut = dict()
for u in Es.users():
if not u.got_unix_user:
continue
ulut[u._id] = u
ret['users'][str(u.name)] = {
'full_name': u.full_name,
'expire_date': DT_MIN.strftime('%Y-%m-%d')}
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
ret['users'][str(ulut[user_id].name)]['expire_date'] \
= latest['until'].strftime('%Y-%m-%d')
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the return map
for g in gs:
if not g.got_unix_group:
continue
ret['groups'][str(g.name)] = [str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut]
return ret
def generate_unix_map(giedo):
ret = {'groups': {},
'users': {}}
dt_now = now()
# Get all users
ulut = dict()
for u in Es.users():
if not u.got_unix_user:
continue
ulut[u._id] = u
ret['users'][str(u.name)] = {
'full_name': u.full_name,
'expire_date': DT_MIN.strftime('%Y-%m-%d')}
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
ret['users'][str(ulut[user_id].name)]['expire_date'] \
= latest['until'].strftime('%Y-%m-%d')
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the return map
for g in gs:
if not g.got_unix_group:
continue
ret['groups'][str(g.name)] = [str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut]
return ret
def main():
now = Es.now()
for n, r in enumerate(sorted(Es.query_relations(
_with=Es.id_by_name('leden'), _from=now, until=now,
deref_who=True),
key=lambda r: r['from'])):
print n+1, r['who'].humanName
def generate_postfix_map(giedo):
tbl = dict() # the virtual map
non_mailman_groups = {}
dt_now = now()
id2email = {}
# handle the straightforward cases
for e in Es.all():
if e.canonical_email is None:
continue
id2email[e._id] = e.canonical_email
for nm in e.other_names:
tbl["%s@%s" % (nm, settings.MAILDOMAIN)] = (e.canonical_email,)
if e.type == 'user':
tbl[e.canonical_email] = (e.email,)
elif e.type == 'group':
if e.got_mailman_list and e.name:
tbl[e.canonical_email] = ('%s@%s' % (
str(e.name), settings.LISTS_MAILDOMAIN),)
else:
tbl[e.canonical_email] = []
non_mailman_groups[e._id] = e
else:
logging.warn("postfix: unhandled type: %s" % e.type)
id_email = "%s@%s" % (e.id, settings.MAILDOMAIN)
if id_email not in tbl:
tbl[id_email] = (e.canonical_email,)
# handle the non-mailman groups
for rel in Es.query_relations(_with=list(non_mailman_groups),
_from=dt_now, until=dt_now, how=None):
e = non_mailman_groups[rel['with']]
email = id2email.get(rel['who'])
if email is not None:
tbl[e.canonical_email].append(email)
return tbl
def relation_begin(request):
# TODO We should use Django forms, or better: use sweet Ajax
d = {}
for t in ('who', 'with', 'how'):
if t not in request.POST:
raise ValueError, "Missing attr %s" % t
if t == 'how' and request.POST[t] == 'null':
d[t] = None
else:
d[t] = _id(request.POST[t])
if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
d['how']):
raise PermissionDenied
# Check whether such a relation already exists
dt = now()
ok = False
try:
next(Es.query_relations(who=d['who'], _with=d['with'],
how=d['how'], _from=dt, until=DT_MAX))
except StopIteration:
ok = True
if not ok:
raise ValueError, "This relation already exists"
# Add the relation!
Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)
giedo.sync()
return redirect_to_referer(request)
def main2():
rels = list(Es.query_relations(-1,
Es.by_name('comms').get_bearers(),
None,
now(),
deref_who=True,
deref_with=True))
lut = {}
for rel in rels:
if rel['from'] is None:
rel['from'] = Es.DT_MIN
if not rel['with'] in lut:
lut[rel['with']] = {}
v = (now() - rel['from']).days / 365.0
if rel['who'] not in lut[rel['with']] or \
lut[rel['with']][rel['who']] > v:
lut[rel['with']][rel['who']] = v
pairs = lut.items()
for comm, members in pairs:
print(six.text_type(comm.humanName))
mpairs = members.items()
mpairs.sort(key=lambda x: x[1])
for member, time in mpairs:
print(' %-20s%.2f' % (member.name, time))
def main2():
rels = list(
Es.query_relations(-1,
Es.by_name('comms').get_bearers(),
None,
now(),
deref_who=True,
deref_with=True))
lut = {}
for rel in rels:
if rel['from'] is None:
rel['from'] = Es.DT_MIN
if not rel['with'] in lut:
lut[rel['with']] = {}
v = (now() - rel['from']).days / 365.0
if rel['who'] not in lut[rel['with']] or \
lut[rel['with']][rel['who']] > v:
lut[rel['with']][rel['who']] = v
pairs = lut.items()
for comm, members in pairs:
print(six.text_type(comm.humanName))
mpairs = members.items()
mpairs.sort(key=lambda x: x[1])
for member, time in mpairs:
print(' %-20s%.2f' % (member.name, time))
def main():
now = Es.now()
member_age = {}
for rel in Es.query_relations(-1,
Es.by_name('leden'),
None,
None,
deref_who=True):
if not rel['from']:
rel['from'] = Es.DT_MIN
if rel['who'] not in member_age:
member_age[rel['who']] = Es.DT_MAX
member_age[rel['who']] = rel['from']
l = []
for m, age in member_age.iteritems():
if not m.dateOfBirth:
continue
print(age - m.dateOfBirth).days / 365.242, unicode(m.name)
l.append((age - m.dateOfBirth).days / 365.242)
print 'avg', sum(l) / len(l)
print 'med', sorted(l)[len(l) / 2]
print '1st', sorted(l)[len(l) / 4 * 2]
print '3rd', sorted(l)[len(l) / 4 * 3]
print 'min', min(l)
print 'max', max(l)
def generate_postfix_map(giedo):
tbl = dict() # the virtual map
non_mailman_groups = {}
dt_now = now()
id2email = {}
# handle the straightforward cases
for e in Es.all():
if e.canonical_email is None or e.name is None:
continue
id2email[e._id] = e.canonical_email
for nm in e.other_names:
tbl["%s@%s" % (nm, MAILDOMAIN)] = (e.canonical_email,)
if e.type == 'user':
tbl[e.canonical_email] = (e.primary_email,)
elif e.type == 'group':
if e.got_mailman_list:
tbl[e.canonical_email] = ('%s@%s' % (
str(e.name), LISTS_MAILDOMAIN),)
else:
tbl[e.canonical_email] = []
non_mailman_groups[e._id] = e
else:
logging.warn("postfix: unhandled type: %s" % e.type)
# handle the non-mailman groups
for rel in Es.query_relations(_with=non_mailman_groups.keys(),
_from=dt_now, until=dt_now, how=None):
e = non_mailman_groups[rel['with']]
email = id2email.get(rel['who'])
if email is not None:
tbl[e.canonical_email].append(email)
return tbl
def relation_begin(request):
# TODO We should use Django forms, or better: use sweet Ajax
d = {}
for t in ("who", "with", "how"):
if t not in request.POST:
raise ValueError, "Missing attr %s" % t
if t == "how" and (not request.POST[t] or request.POST[t] == "null"):
d[t] = None
else:
d[t] = _id(request.POST[t])
if not Es.user_may_begin_relation(request.user, d["who"], d["with"], d["how"]):
raise PermissionDenied
# Check whether such a relation already exists
dt = now()
ok = False
try:
next(Es.query_relations(who=d["who"], _with=d["with"], how=d["how"], _from=dt, until=DT_MAX))
except StopIteration:
ok = True
if not ok:
raise ValueError, _("Deze relatie bestaat al")
# Add the relation!
relation_id = Es.add_relation(d["who"], d["with"], d["how"], dt, DT_MAX)
# Notify informacie
# TODO (rik) leave out 'als lid'
Es.notify_informacie("relation_begin", request.user, relation=relation_id)
giedo.sync_async(request)
return redirect_to_referer(request)
def generate_postfix_slm_map(giedo):
# We generate the postfix "sender_login_maps".
# This is used to decide by postfix whether a given user is allowed to
# send e-mail as if it was coming from a particular e-mail address.
# It is a dictionary { <email address> : [ <allowed user>, ... ] }
tbl = dict()
dt_now = now()
# Get all users
ulut = dict()
# We only allow members to send e-mail
for u in Es.by_name('leden').get_members():
ulut[u._id] = u
for name in u.names:
if str(name) not in tbl:
tbl[str(name)] = set()
tbl[str(name)].add(str(u.name))
# There are two kind of groups: groups whose members are allowed to
# send e-mail as if coming from the group itself and those where this
# is not allowed. For convenience, lets call the first kind the
# impersonatable groups.
# Get all impersonatable groups and create a look-up-table for
# group membership
gs = list()
for g in Es.groups():
# TODO add a tag to force a group either impersonatable or not
if not g.got_mailman_list:
gs.append(g)
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a impersonatable group
# and a non-group subgraph.
mlut_g = {} # { <group> : <members that are impersonatable groups> }
mlut_u = {} # { <group> : <members that are users> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_u[g_id] = [c for c in mlut[g_id] if c in ulut]
mlut_g_tc = tc(mlut_g) # transitive closure
for g in gs:
to_consider = tuple(mlut_g_tc[g._id]) + (g._id,)
for sg_id in to_consider:
for u_id in mlut_u[sg_id]:
for name in g.names:
if str(name) not in tbl:
tbl[str(name)] = set()
tbl[str(name)].add(str(ulut[u_id].name))
# Clean up tbl to return.
ret = {}
for name, users in six.iteritems(tbl):
if not users:
continue
ret["%s@%s" % (name, settings.MAILDOMAIN)] = tuple(users)
return ret
def main():
now = Es.now()
for n, r in enumerate(
sorted(Es.query_relations(_with=Es.id_by_name('leden'),
_from=now,
until=now,
deref_who=True),
key=lambda r: r['from'])):
print(n + 1, r['who'].humanName)
def generate_mailman_changes(giedo):
todo = {'create': [], 'add': {}, 'remove': {}}
# TODO mm_groups and mm_rels
# Get the groups that need a mailing list and the members of those
# groups.
dt_now = now()
mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
mm_rels = Es.query_relations(_with=mm_groups,
how=None,
_from=dt_now,
until=dt_now,
deref_who=True)
# Get the current mailing list membersip
ml_membership = giedo.hans.send({'type': 'maillist-get-membership'})
# membership_to_check will contain the current membership of the
# mailinglists for which there are groups. We will remove users
# from membership_to_check if we see these users are in the groups.
# In the end membership_to_check contains the stray users.
membership_to_check = {}
gid2name = dict()
# Find the current members of the mailing lists and find which
# mailing lists are missing.
for g in mm_groups:
name = str(g.name).encode()
gid2name[g._id] = name
if name in ml_membership:
membership_to_check[name] = set(ml_membership[name])
else:
todo['create'].append((name, six.text_type(g.humanName)))
membership_to_check[name] = set()
# Check which memberships are missing in the current mailing lists
for rel in mm_rels:
em = rel['who'].canonical_email.encode()
gname = gid2name[rel['with']]
if em not in membership_to_check[gname]:
if gname not in todo['add']:
todo['add'][gname] = []
todo['add'][gname].append(em)
else:
membership_to_check[gname].remove(em)
# Check which memberships are superfluous in the current mailing lists
for n in ml_membership:
if n not in membership_to_check:
logging.warning("Unaccounted e-maillist %s" % n)
continue
for em in membership_to_check[n]:
if n not in todo['remove']:
todo['remove'][n] = []
todo['remove'][n].append(em)
return todo
def _brand_detail(request, brand):
ctx = _entity_detail(request, brand)
ctx['rels'] = sorted(
Es.query_relations(how=brand, deref_who=True, deref_with=True),
key=lambda x: (Es.DT_MIN - Es.relation_until(
x), Es.entity_humanName(x['with']), Es.entity_humanName(x['who'])))
for r in ctx['rels']:
r['id'] = r['_id']
r['until_year'] = (None if r['until'] is None else Es.date_to_year(
r['until']))
r['virtual'] = Es.relation_is_virtual(r)
return render(request, 'leden/brand_detail.html', ctx)
def _brand_detail(request, brand):
ctx = _entity_detail(request, brand)
ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
deref_with=True),
key=lambda x: (Es.DT_MIN - Es.relation_until(x),
Es.entity_humanName(x['with']),
Es.entity_humanName(x['who'])))
for r in ctx['rels']:
r['id'] = r['_id']
r['until_year'] = (None if r['until'] is None else
Es.date_to_year(r['until']))
r['virtual'] = Es.relation_is_virtual(r)
return render(request, 'leden/brand_detail.html', ctx)
def qrel(who=-1, _with=-1, how=-1, _from=None, until=None):
""" Queries relations """
if who not in (None, -1):
who = Es.id_by_name(who)
if _with not in (None, -1):
_with = Es.id_by_name(_with)
if how not in (None, -1):
how = Es.ecol.find_one({'sofa_suffix': how})['_id']
if _from not in (None, -1):
_from = str_to_date(_from)
if until not in (None, -1):
until = str_to_date(until)
return list(Es.query_relations(who, _with, how, _from, until, True,
True, True))
def qrel(who=-1, _with=-1, how=-1, _from=None, until=None):
""" Queries relations """
if who not in (None, -1):
who = Es.id_by_name(who)
if _with not in (None, -1):
_with = Es.id_by_name(_with)
if how not in (None, -1):
how = Es.ecol.find_one({'sofa_suffix': how})['_id']
if _from not in (None, -1):
_from = str_to_date(_from)
if until not in (None, -1):
until = str_to_date(until)
return list(
Es.query_relations(who, _with, how, _from, until, True, True, True))
def _group_detail(request, group):
ctx = _entity_detail(request, group)
isFreeToJoin = group.has_tag(Es.id_by_name("!free-to-join", True))
rel_id = None
if isFreeToJoin:
dt = now()
rel = list(Es.query_relations(request.user, group, None, dt, dt, False, False, False))
assert len(rel) <= 1
for r in rel:
rel_id = r["_id"]
ctx.update(
{"isFreeToJoin": group.has_tag(Es.by_name("!free-to-join")), "request": request, "relation_with_group": rel_id}
)
return render_to_response("leden/group_detail.html", ctx, context_instance=RequestContext(request))
def _sync_villanet(self):
ret = self.villanet_request({'action': 'listUsers'})
if not ret[0]:
return
ret = json.loads(ret[1])
users = dict()
ulut = dict()
for u in Es.users():
ulut[u._id] = str(u.name)
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=now()):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
users[ulut[user_id]] = latest['until'].strftime('%Y-%m-%d')
vn = set(ret.keys())
kn = set(users.keys())
dt_max = settings.DT_MAX.strftime('%Y-%m-%d')
for name in kn - vn:
data = {
'username': name,
'password': self.villanet_encrypt_password(pseudo_randstr(16)),
}
if users[name] != dt_max:
data['till'] = users[name]
pc = Es.PushChange({
'system': 'villanet',
'action': 'addUser',
'data': data
})
pc.save()
for name in vn - kn:
logging.info("Stray user %s" % name)
for name in vn & kn:
remote = (ret[name]['till'][:10]
if ret[name]['till'] is not None else '')
local = users[name] if users[name] != dt_max else ''
if remote != local:
pc = Es.PushChange({
'system': 'villanet',
'action': 'changeUser',
'data': {
'username': name,
'till': local
}
})
pc.save()
self.push_changes_event.set()
def generate_mailman_changes(giedo):
todo = {'create': [], 'add': {}, 'remove': {}}
# TODO mm_groups and mm_rels
# Get the groups that need a mailing list and the members of those
# groups.
dt_now = now()
mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
until=dt_now, deref_who=True)
# Get the current mailing list membersip
ml_membership = giedo.hans.send({'type': 'maillist-get-membership'})
# membership_to_check will contain the current membership of the
# mailinglists for which there are groups. We will remove users
# from membership_to_check if we see these users are in the groups.
# In the end membership_to_check contains the stray users.
membership_to_check = {}
gid2name = dict()
# Find the current members of the mailing lists and find which
# mailing lists are missing.
for g in mm_groups:
name = str(g.name).encode()
gid2name[g._id] = name
if name in ml_membership:
membership_to_check[name] = set(ml_membership[name])
else:
todo['create'].append((name, six.text_type(g.humanName)))
membership_to_check[name] = set()
# Check which memberships are missing in the current mailing lists
for rel in mm_rels:
em = rel['who'].canonical_email.encode()
gname = gid2name[rel['with']]
if em not in membership_to_check[gname]:
if gname not in todo['add']:
todo['add'][gname] = []
todo['add'][gname].append(em)
else:
membership_to_check[gname].remove(em)
# Check which memberships are superfluous in the current mailing lists
for n in ml_membership:
if n not in membership_to_check:
logging.warning("Unaccounted e-maillist %s" % n)
continue
for em in membership_to_check[n]:
if n not in todo['remove']:
todo['remove'][n] = []
todo['remove'][n].append(em)
return todo
def _group_detail(request, group):
ctx = _entity_detail(request, group)
isFreeToJoin = group.has_tag(Es.id_by_name('!free-to-join', True))
rel_id = None
if isFreeToJoin:
dt = now()
rel = list(Es.query_relations(request.user, group, None,
dt, dt, False, False, False))
assert len(rel) <= 1
for r in rel:
rel_id = r['_id']
ctx.update({'isFreeToJoin': group.has_tag(Es.by_name('!free-to-join')),
'request': request,
'relation_with_group': rel_id})
return render(request, 'leden/group_detail.html', ctx)
def generate_mailman_changes(hans):
todo = hans_pb2.ApplyChangesReq()
# TODO mm_groups and mm_rels
# Get the groups that need a mailing list and the members of those
# groups.
dt_now = now()
mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
until=dt_now, deref_who=True)
# Get the current mailing list membersip
ml_membership = hans.GetMembership(hans_pb2.GetMembershipReq()).membership
# membership_to_check will contain the current membership of the
# mailinglists for which there are groups. We will remove users
# from membership_to_check if we see these users are in the groups.
# In the end membership_to_check contains the stray users.
membership_to_check = {}
gid2name = dict()
# Find the current members of the mailing lists and find which
# mailing lists are missing.
for g in mm_groups:
name = str(g.name).encode()
gid2name[g._id] = name
if name in ml_membership:
membership_to_check[name] = set(ml_membership[name].emails)
else:
todo.create.append(hans_pb2.ListCreateReq(
name=name,
humanName=six.text_type(g.humanName)))
membership_to_check[name] = set()
# Check which memberships are missing in the current mailing lists
for rel in mm_rels:
em = rel['who'].canonical_email.encode()
gname = gid2name[rel['with']]
if em not in membership_to_check[gname]:
todo.add[gname].emails.append(em)
else:
membership_to_check[gname].remove(em)
# Check which memberships are superfluous in the current mailing lists
for n in ml_membership:
if n not in membership_to_check:
logging.warning("Unaccounted e-maillist %s" % n)
continue
for em in membership_to_check[n]:
todo.remove[n].emails.append(em)
return todo
def _sync_villanet(self):
if not settings.VILLANET_SECRET_API_KEY:
logging.warn("VILLANET_SECRET_API_KEY not set")
return
ret = self.villanet_request({'action': 'listUsers'})
if not ret[0]:
return
ret = json.loads(ret[1])
users = dict()
ulut = dict()
for u in Es.users():
ulut[u._id] = str(u.name)
member_relations_grouped = dict()
for rel in Es.query_relations(_with=Es.by_name('leden'), until=now()):
if rel['who'] not in member_relations_grouped:
member_relations_grouped[rel['who']] = []
member_relations_grouped[rel['who']].append(rel)
for user_id, relations in member_relations_grouped.items():
latest = max(relations, key=lambda x: x['until'])
users[ulut[user_id]] = latest['until'].strftime('%Y-%m-%d')
vn = set(ret.keys())
kn = set(users.keys())
dt_max = settings.DT_MAX.strftime('%Y-%m-%d')
for name in kn - vn:
data = {
'username': name,
'password': self.villanet_encrypt_password(
pseudo_randstr(16)),
}
if users[name] != dt_max:
data['till'] = users[name]
pc = Es.PushChange({'system': 'villanet', 'action': 'addUser',
'data': data})
pc.save()
for name in vn - kn:
logging.info("Stray user %s" % name)
for name in vn & kn:
remote = (ret[name]['till'][:10] if ret[name]['till'] is not None
else '')
local = users[name] if users[name] != dt_max else ''
if remote != local:
pc = Es.PushChange({'system': 'villanet',
'action': 'changeUser', 'data': {
'username': name,
'till': local
}})
pc.save()
self.push_changes_event.set()
def _group_detail(request, group):
ctx = _entity_detail(request, group)
isFreeToJoin = group.has_tag(Es.id_by_name('!free-to-join', True))
rel_id = None
if isFreeToJoin:
dt = now()
rel = list(Es.query_relations(request.user, group, None,
dt, dt, False, False, False))
assert len(rel) <= 1
for r in rel:
rel_id = r['_id']
ctx.update({'isFreeToJoin': group.has_tag(Es.by_name('!free-to-join')),
'request': request,
'relation_with_group': rel_id})
return render_to_response('leden/group_detail.html', ctx,
context_instance=RequestContext(request))
def generate_mailman_changes(giedo):
todo = {'create': [], 'add': {}, 'remove': {}}
# TODO mm_groups and mm_rels
# Get the groups that need a mailing list and the members of those
# groups.
dt_now = now()
mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
until=dt_now, deref_who=True)
# TODO do we want to cache these?
# Get the current mailing lists
ml_names = frozenset(Mailman.Utils.list_names())
ml_members = dict()
gid2name = dict()
# Find the current members of the mailing lists and find which
# mailing lists are missing.
for g in mm_groups:
gid2name[g._id] = str(g.name)
if not str(g.name) in ml_names:
todo['create'].append((str(g.name),
unicode(g.humanName)))
c_ms = set([])
else:
c_ms = set([x[0] for x in
Mailman.MailList.MailList(str(g.name),
lock=False).members.iteritems()])
ml_members[str(g.name)] = c_ms
# Check which memberships are missing in the current mailing lists
for rel in mm_rels:
em = rel['who'].canonical_email
gname = gid2name[rel['with']]
if not em in ml_members[gname]:
if not gname in todo['add']:
todo['add'][gname] = []
todo['add'][gname].append(em)
else:
ml_members[gname].remove(em)
# Check which memberships are superfluous in the current mailing lists
for n in ml_names:
if not n in ml_members:
logging.warning("Unaccounted e-maillist %s" % n)
continue
for em in ml_members[n]:
if not n in todo['remove']:
todo['remove'][n] = []
todo['remove'][n].append(em)
return todo
def main3():
member_age = {}
for rel in Es.query_relations(-1, Es.by_name('leden'), None,
None, deref_who=True):
if rel['who'] not in member_age:
member_age[rel['who']] = 0
member_age[rel['who']] = max(member_age[rel['who']],
(now() - rel['from']).days / 365.0)
# for comm in Es.by_name('comms').get_bearers():
for comm in [Es.by_name('draai')]:
print(six.text_type(comm.humanName))
members = [(m, member_age.get(m)) for m in comm.get_members()]
members.sort(key=lambda x: x[1])
for member in members:
print(" %-20s%.2f" % (six.text_type(member[0].name),
member[1] if member[1] else -1))
def main():
year_overrides = dict()
had = set()
for t in Es.bearers_by_tag_id(
Es.id_by_name('!year-overrides'),
_as=Es.Tag):
year_overrides[(t._data['year-override']['type'],
t._data['year-override']['year'])] = t._data['_id']
for rel in sorted(Es.query_relations(-1, Es.id_by_name('leden'), None,
None, None, True, False, False),
key=lambda x: x['from']):
name = str(rel['who'].name)
if name in had:
continue
if ('temp' not in rel['who']._data or
'joined' not in rel['who']._data['temp']):
had.add(name)
continue
joined = datetime.datetime.strptime(
rel['who']._data['temp']['joined'], '%Y-%m-%d')
if joined == rel['from']:
had.add(name)
continue
joined_yr = Es.date_to_year(joined)
from_yr = Es.date_to_year(rel['from'])
if joined_yr == from_yr or joined_yr == 0:
print(name, rel['from'].date(), ' -> ',
joined.date())
rrel = Es.rcol.find({'_id': rel['_id']})[0]
rrel['from'] = joined
Es.rcol.save(rrel)
had.add(name)
continue
if joined_yr == from_yr - 1:
print(name, rel['from'].date(), ' -> ',
joined.date())
rrel = Es.rcol.find({'_id': rel['_id']})[0]
rrel['from'] = joined
if 'tags' not in rrel:
rrel['tags'] = []
if not year_overrides[(False, joined_yr)] in rrel['tags']:
rrel['tags'].append(year_overrides[(False, joined_yr)])
Es.rcol.save(rrel)
had.add(name)
continue
print('MANUAL REVIEW: ', name, joined.date(), rel['from'].date())
def main():
year_overrides = dict()
had = set()
for t in Es.bearers_by_tag_id(Es.id_by_name('!year-overrides'),
_as=Es.Tag):
year_overrides[(t._data['year-override']['type'],
t._data['year-override']['year'])] = t._data['_id']
for rel in sorted(Es.query_relations(-1, Es.id_by_name('leden'), None,
None, None, True, False, False),
cmp=lambda x, y: cmp(x['from'], y['from'])):
name = str(rel['who'].name)
if name in had:
continue
if ('temp' not in rel['who']._data
or 'joined' not in rel['who']._data['temp']):
had.add(name)
continue
joined = datetime.datetime.strptime(rel['who']._data['temp']['joined'],
'%Y-%m-%d')
if joined == rel['from']:
had.add(name)
continue
joined_yr = Es.date_to_year(joined)
from_yr = Es.date_to_year(rel['from'])
if joined_yr == from_yr or joined_yr == 0:
print name, rel['from'].date(), ' -> ', \
joined.date()
rrel = Es.rcol.find({'_id': rel['_id']})[0]
rrel['from'] = joined
Es.rcol.save(rrel)
had.add(name)
continue
if joined_yr == from_yr - 1:
print name, rel['from'].date(), ' -> ', \
joined.date()
rrel = Es.rcol.find({'_id': rel['_id']})[0]
rrel['from'] = joined
if not 'tags' in rrel:
rrel['tags'] = []
if not year_overrides[(False, joined_yr)] in rrel['tags']:
rrel['tags'].append(year_overrides[(False, joined_yr)])
Es.rcol.save(rrel)
had.add(name)
continue
print 'MANUAL REVIEW: ', name, \
joined.date(), rel['from'].date()
def main3():
member_age = {}
for rel in Es.query_relations(-1,
Es.by_name('leden'),
None,
None,
deref_who=True):
if rel['who'] not in member_age:
member_age[rel['who']] = 0
member_age[rel['who']] = max(member_age[rel['who']],
(now() - rel['from']).days / 365.0)
# for comm in Es.by_name('comms').get_bearers():
for comm in [Es.by_name('draai')]:
print(six.text_type(comm.humanName))
members = [(m, member_age.get(m)) for m in comm.get_members()]
members.sort(key=lambda x: x[1])
for member in members:
print(" %-20s%.2f" % (six.text_type(
member[0].name), member[1] if member[1] else -1))
def _brand_detail(request, brand):
ctx = _entity_detail(request, brand)
def _cmp(x, y):
r = Es.relation_cmp_until(y, x)
if r:
return r
r = cmp(unicode(x["with"].humanName), unicode(y["with"].humanName))
if r:
return r
r = cmp(unicode(x["who"].humanName), unicode(y["who"].humanName))
if r:
return r
return Es.relation_cmp_from(x, y)
ctx["rels"] = sorted(Es.query_relations(how=brand, deref_who=True, deref_with=True), cmp=_cmp)
for r in ctx["rels"]:
r["id"] = r["_id"]
r["until_year"] = None if r["until"] is None else Es.date_to_year(r["until"])
r["virtual"] = Es.relation_is_virtual(r)
return render_to_response("leden/brand_detail.html", ctx, context_instance=RequestContext(request))
def _brand_detail(request, brand):
ctx = _entity_detail(request, brand)
def _cmp(x,y):
r = Es.relation_cmp_until(y,x)
if r: return r
r = cmp(unicode(x['with'].humanName),
unicode(y['with'].humanName))
if r: return r
r = cmp(unicode(x['who'].humanName),
unicode(y['who'].humanName))
if r: return r
return Es.relation_cmp_from(x,y)
ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
deref_with=True), cmp=_cmp)
for r in ctx['rels']:
r['id'] = r['_id']
r['until_year'] = (None if r['until'] is None else
Es.date_to_year(r['until']))
r['virtual'] = Es.relation_is_virtual(r)
return render_to_response('leden/brand_detail.html', ctx,
context_instance=RequestContext(request))
def _brand_detail(request, brand):
ctx = _entity_detail(request, brand)
def _cmp(x,y):
r = Es.relation_cmp_until(y,x)
if r: return r
r = cmp(unicode(x['with'].humanName),
unicode(y['with'].humanName))
if r: return r
r = cmp(unicode(x['who'].humanName),
unicode(y['who'].humanName))
if r: return r
return Es.relation_cmp_from(x,y)
ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
deref_with=True), cmp=_cmp)
for r in ctx['rels']:
r['id'] = r['_id']
r['until_year'] = (None if r['until'] is None else
Es.date_to_year(r['until']))
r['virtual'] = Es.relation_is_virtual(r)
return render_to_response('leden/brand_detail.html', ctx,
context_instance=RequestContext(request))
def main():
member_age = {}
for rel in Es.query_relations(-1, Es.by_name('leden'), None,
None, deref_who=True):
if not rel['from']:
rel['from'] = Es.DT_MIN
if rel['who'] not in member_age:
member_age[rel['who']] = Es.DT_MAX
member_age[rel['who']] = rel['from']
le = []
for m, age in six.iteritems(member_age):
if not m.dateOfBirth:
continue
print((age - m.dateOfBirth).days / 365.242, six.text_type(m.name))
le.append((age - m.dateOfBirth).days / 365.242)
print('avg', sum(le) / len(le))
print('med', sorted(le)[len(le) / 2])
print('1st', sorted(le)[len(le) / 4 * 2])
print('3rd', sorted(le)[len(le) / 4 * 3])
print('min', min(le))
print('max', max(le))
def _generate_member_count():
events = []
for rel in Es.query_relations(_with=Es.id_by_name('leden'), how=None):
events.append((max(rel['from'], Es.DT_MIN), True))
if rel['until'] != Es.DT_MAX:
events.append((rel['until'], False))
N = 0
old_days = -1
old_N = None
ret = []
for when, what in sorted(events, key=lambda x: x[0]):
N += 1 if what else -1
days = (when - Es.DT_MIN).days
if old_days != days:
if old_N:
ret.append([old_days, old_N])
old_days = days
old_N = N
ret.append([days, N])
ret = [(1 + days2 / 365.242, N2) for days2, N2 in ret]
return ret
def _generate_member_count():
events = []
for rel in Es.query_relations(_with=Es.id_by_name('leden'), how=None):
events.append((max(rel['from'], Es.DT_MIN), True))
if rel['until'] != Es.DT_MAX:
events.append((rel['until'], False))
N = 0
old_days = -1
old_N = None
ret = []
for when, what in sorted(events, key=lambda x: x[0]):
N += 1 if what else -1
days = (when - Es.DT_MIN).days
if old_days != days:
if old_N:
ret.append([old_days, old_N])
old_days = days
old_N = N
ret.append([days, N])
ret = [(1 + days / 365.242, N) for days, N in ret]
return ret
def relation_begin(request):
# TODO We should use Django forms, or better: use sweet Ajax
d = {}
for t in ('who', 'with', 'how'):
if t not in request.POST:
raise ValueError, "Missing attr %s" % t
if t == 'how' and (not request.POST[t] or
request.POST[t] == 'null'):
d[t] = None
else:
d[t] = _id(request.POST[t])
if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
d['how']):
raise PermissionDenied
# Check whether such a relation already exists
dt = now()
ok = False
try:
next(Es.query_relations(who=d['who'], _with=d['with'],
how=d['how'], _from=dt, until=DT_MAX))
except StopIteration:
ok = True
if not ok:
raise ValueError, "This relation already exists"
# Add the relation!
Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)
# Notify informacie
if request.user._id == d['who']:
Es.notify_informacie("%s heeft zich ingeschreven als %s %s" % (
request.user.full_name,
Es.by_id(d['how']).humanName if d['how'] else 'lid',
Es.by_id(d['with']).humanName.genitive))
else:
# TODO (rik) leave out 'als lid'
Es.notify_informacie("%s is nu %s %s" % (
Es.by_id(d['who']).humanName,
Es.by_id(d['how']).humanName if d['how'] else 'lid',
Es.by_id(d['with']).humanName.genitive))
giedo.sync_async(request)
return redirect_to_referer(request)
def relation_begin(request):
# TODO We should use Django forms, or better: use sweet Ajax
d = {}
for t in ('who', 'with', 'how'):
if t not in request.POST:
raise ValueError("Missing attr %s" % t)
if t == 'how' and (not request.POST[t] or request.POST[t] == 'null'):
d[t] = None
else:
d[t] = _id(request.POST[t])
if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
d['how']):
raise PermissionDenied
# Check whether such a relation already exists
dt = now()
ok = False
try:
next(
Es.query_relations(who=d['who'],
_with=d['with'],
how=d['how'],
_from=dt,
until=DT_MAX))
except StopIteration:
ok = True
if not ok:
messages.info(request, _("Deze relatie bestaat al"))
return redirect_to_referer(request)
# Add the relation!
relation_id = Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)
# Notify informacie
# TODO (rik) leave out 'als lid'
Es.notify_informacie('relation_begin', request.user, relation=relation_id)
giedo.sync_async(request)
return redirect_to_referer(request)
def main():
now = Es.now()
member_age = {}
for rel in Es.query_relations(-1, Es.by_name('leden'), None,
None, deref_who=True):
if not rel['from']:
rel['from'] = Es.DT_MIN
if rel['who'] not in member_age:
member_age[rel['who']] = Es.DT_MAX
member_age[rel['who']] = rel['from']
l = []
for m, age in member_age.iteritems():
if not m.dateOfBirth:
continue
print (age - m.dateOfBirth).days / 365.242, unicode(m.name)
l.append( (age - m.dateOfBirth).days / 365.242)
print 'avg', sum(l)/len(l)
print 'med', sorted(l)[len(l)/2]
print '1st', sorted(l)[len(l)/4*2]
print '3rd', sorted(l)[len(l)/4*3]
print 'min', min(l)
print 'max', max(l)
def main():
member_age = {}
for rel in Es.query_relations(-1,
Es.by_name('leden'),
None,
None,
deref_who=True):
if not rel['from']:
rel['from'] = Es.DT_MIN
if rel['who'] not in member_age:
member_age[rel['who']] = Es.DT_MAX
member_age[rel['who']] = rel['from']
le = []
for m, age in six.iteritems(member_age):
if not m.dateOfBirth:
continue
print((age - m.dateOfBirth).days / 365.242, six.text_type(m.name))
le.append((age - m.dateOfBirth).days / 365.242)
print('avg', sum(le) / len(le))
print('med', sorted(le)[len(le) / 2])
print('1st', sorted(le)[len(le) / 4 * 2])
print('3rd', sorted(le)[len(le) / 4 * 3])
print('min', min(le))
print('max', max(le))
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(
('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(six.itervalues(groups))
# Find groups that have a virtual group for each year
year_groups = [g for g in groups_set if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
years_of_year_overrides = [
yo[1] for yo in six.itervalues(year_overrides)
]
until_years = [
Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX
]
max_until = max(
max(until_years) if until_years else 0,
max(years_of_year_overrides) if years_of_year_overrides else 0,
Es.date_to_year(dt_now))
from_years = [
Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN
]
min_from = min(
min(from_years) if from_years else Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = (min_from
if rel['from'] == DT_MIN else Es.date_to_year(rel['from']))
t = (max_until
if rel['until'] == DT_MAX else Es.date_to_year(rel['until']))
years = set(range(s, t + 1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
tp, yr = year_overrides[tid]
if tp:
years.add(yr)
else:
if yr not in years:
logging.warning('bogus year-override -{} on {}'.format(
yr, rel['_id']))
continue
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups, id2name)
# Find all virtual groups
virtual_groups = [
g for g in groups_set if tags['!virtual-group'] in g.tag_ids
]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s" %
vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'], 'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
# If there are two relations between an entity and a group in the
# same year, we do not want this relation to be handled twice.
# Thus we keep a seperate look-up-table to prevent this.
year_vgroup_rel_ok = set()
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {
'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX
}
if (not relkey(rrel) in vgroup_rlut
and relkey(rrel) not in year_vgroup_rel_ok):
logging.info(
"vgroup: adding %s -> %s (%s)" %
(id2name[mrel['who']], yg.name, id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
elif relkey(rrel) in vgroup_rlut:
del vgroup_rlut[relkey(rrel)]
year_vgroup_rel_ok.add(relkey(rrel))
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=tuple(sofa_brands.values())):
if (rel['how'], rel['with']) in sofa_lut:
continue
if not id2name[rel['with']]:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {
'types': ['group', 'tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']
},
'humanNames': [{
'name':
nm,
'human':
six.text_type(g.humanName) + ' ' +
six.text_type(sofa_brands[rel['how']].humanName)
}]
}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'], 'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {
'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id
}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" %
(id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" %
(id2name[relkey[0]], id2name.get(
relkey[2]), id2name.get(relkey[1])))
Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
relkey[4])
# Set is_active on Users if and only if they are not in the `leden' group.
# TODO We might optimize this by including it in a more generic process
active_users = [
rel['who'] for rel in Es.by_name('leden').get_rrelated(
None, dt_now, dt_now, False, False, False)
]
for u in Es.users():
is_active = u._id in active_users
if u.is_active == is_active:
continue
u._data['is_active'] = is_active
u.save()
logging.info("%s user %s",
("activated" if is_active else "deactivated"),
str(u.name))
def generate_wolk_changes(giedo):
creds = settings.WOLK_MYSQL_SECRET
if not creds:
logging.warning('wolk: no credentials available, skipping')
return None
todo = {
'addUser': [],
'addGroup': [],
'addUserToGroup': [],
'removeUserFromGroup': []
}
dt_now = now()
# First, lets see which users and groups to create
users = dict() # users to have
groups = dict() # groups to have
missing_groups = set()
missing_users = set()
ulut = dict()
for m in Es.by_name('leden').get_members():
if not m.got_unix_user:
continue
ulut[m._id] = m
users[str(m.name)] = m
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the groups variable
for g in gs:
if not g.got_unix_group:
continue
groups[str(g.name)] = set(
[str(ulut[c].name) for c in memb_graph[g._id] if c in ulut])
# Now, check which users and groups actually exist in owncloud
missing_users = set(users.keys())
missing_groups = set(groups.keys())
dc = pymysql.connect(host=creds[0],
user=creds[1],
password=creds[2],
db=creds[3],
charset='utf8')
try:
with dc.cursor() as c:
c.execute("SELECT gid, uid FROM oc_group_user")
for group, user in c.fetchall():
if group not in groups:
continue
if user not in users or user not in groups[group]:
todo['removeUserFromGroup'].append((user, group))
continue
if user in groups[group]:
groups[group].remove(user)
c.execute("SELECT uid FROM oc_users")
for user, in c.fetchall():
if user not in users:
logging.info("wolk: stray user %s", user)
continue
missing_users.remove(user)
c.execute("SELECT gid FROM oc_groups")
for group, in c.fetchall():
if group not in groups:
logging.info("wolk: stray group %s", user)
continue
missing_groups.remove(group)
for user in missing_users:
todo['addUser'].append(
(user, six.text_type(users[user].humanName)))
todo['addGroup'] = list(missing_groups)
for group, missing_members in six.iteritems(groups):
for user in missing_members:
todo['addUserToGroup'].append((user, group))
finally:
dc.close()
return todo
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(('!year-group', '!year-overrides',
'!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(groups.values())
# Find groups that have a virtual group for each year
year_groups = [g for g in groups.itervalues()
if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
until_years = [Es.date_to_year(r['until']) for r in rels
if r['until'] != DT_MAX]
max_until = max(max(until_years) if until_years else 0,
Es.date_to_year(dt_now))
from_years = [Es.date_to_year(r['from']) for r in rels
if r['from'] != DT_MIN]
min_from = min(min(from_years) if from_years else
Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = min_from if rel['from'] == DT_MIN \
else Es.date_to_year(rel['from'])
t = max_until if rel['until'] == DT_MAX \
else Es.date_to_year(rel['until'])
years = set(range(s, t+1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
yr, tp = year_overrides[tid]
if tp:
years.add(yr)
else:
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(
_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = filter(lambda x: x['with']==g._id, year_group_mrels)
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups,
id2name)
# Find all virtual groups
virtual_groups = [g for g in groups_set
if tags['!virtual-group'] in g.tag_ids]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s" \
% vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'],
rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'],
'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX}
if not relkey(rrel) in vgroup_rlut:
logging.info("vgroup: adding %s -> %s (%s)" % (
id2name[mrel['who']], yg.name,
id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=sofa_brands.values()):
if (rel['how'], rel['with']) in sofa_lut:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {'types': ['group','tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']},
'humanNames': [{
'name': nm,
'human': unicode(g.humanName) + ' ' +
unicode(sofa_brands[rel['how']].humanName)}]}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'],
'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" % (
id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" % (
id2name[relkey[0]], id2name.get(relkey[2]),
id2name.get(relkey[1])))
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(('!year-group', '!year-overrides',
'!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(six.itervalues(groups))
# Find groups that have a virtual group for each year
year_groups = [g for g in groups_set
if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
years_of_year_overrides = [yo[1] for yo
in six.itervalues(year_overrides)]
until_years = [Es.date_to_year(r['until']) for r in rels
if r['until'] != DT_MAX]
max_until = max(
max(until_years) if until_years else 0,
max(years_of_year_overrides) if years_of_year_overrides else 0,
Es.date_to_year(dt_now)
)
from_years = [Es.date_to_year(r['from']) for r in rels
if r['from'] != DT_MIN]
min_from = min(min(from_years) if from_years else
Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = (min_from if rel['from'] == DT_MIN
else Es.date_to_year(rel['from']))
t = (max_until if rel['until'] == DT_MAX
else Es.date_to_year(rel['until']))
years = set(range(s, t + 1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
tp, yr = year_overrides[tid]
if tp:
years.add(yr)
else:
if yr not in years:
logging.warning('bogus year-override -{} on {}'.format(
yr, rel['_id']))
continue
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups, id2name)
# Find all virtual groups
virtual_groups = [g for g in groups_set
if tags['!virtual-group'] in g.tag_ids]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s"
% vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'],
rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'],
'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
# If there are two relations between an entity and a group in the
# same year, we do not want this relation to be handled twice.
# Thus we keep a seperate look-up-table to prevent this.
year_vgroup_rel_ok = set()
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX}
if (not relkey(rrel) in vgroup_rlut and relkey(rrel)
not in year_vgroup_rel_ok):
logging.info("vgroup: adding %s -> %s (%s)" % (
id2name[mrel['who']], yg.name,
id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
elif relkey(rrel) in vgroup_rlut:
del vgroup_rlut[relkey(rrel)]
year_vgroup_rel_ok.add(relkey(rrel))
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=tuple(sofa_brands.values())):
if (rel['how'], rel['with']) in sofa_lut:
continue
if not id2name[rel['with']]:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {'types': ['group', 'tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']},
'humanNames': [{
'name': nm,
'human': six.text_type(g.humanName) + ' ' +
six.text_type(sofa_brands[rel['how']].humanName)}]}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'],
'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" % (
id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" % (
id2name[relkey[0]], id2name.get(relkey[2]),
id2name.get(relkey[1])))
Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
relkey[4])
# Set is_active on Users if and only if they are not in the `leden' group.
# TODO We might optimize this by including it in a more generic process
active_users = [rel['who'] for rel in Es.by_name('leden').get_rrelated(
None, dt_now, dt_now, False, False, False)]
for u in Es.users():
is_active = u._id in active_users
if u.is_active == is_active:
continue
u._data['is_active'] = is_active
u.save()
logging.info(
"%s user %s",
("activated" if is_active else "deactivated"),
str(u.name)
)
llut = dict()
clut = dict()
comms = [g for g in Es.by_name('comms').get_bearers() if g.is_group]
comms.append(Es.by_name('bestuur'))
leden = Es.by_name('leden').get_members()
leden_lut = {}
comms_lut = {}
for c in comms:
comms_lut[_id(c)] = c
for l in leden:
clut[l] = (DT_MAX, None)
llut[l] = DT_MAX
leden_lut[_id(l)] = l
for rel in Es.query_relations(who=leden, _with=Es.by_name('leden'), how=None):
w = leden_lut[rel['who']]
llut[w] = min(llut[w], rel['from'])
for rel in Es.query_relations(who=leden, _with=comms, how=None):
w = leden_lut[rel['who']]
if rel['from'] < clut[w][0]:
clut[w] = (rel['from'], comms_lut[rel['with']])
had = set()
for y in range(3, 10):
print()
print('Jaar', y)
nnever = 0
nalways = 0
navg = 0
def generate_wolk_changes(giedo):
creds = settings.WOLK_MYSQL_SECRET
if not creds:
logging.warning('wolk: no credentials available, skipping')
return None
todo = {'addUser': [], 'addGroup': [],
'addUserToGroup': [], 'removeUserFromGroup': []}
dt_now = now()
# First, lets see which users and groups to create
users = dict() # users to have
groups = dict() # groups to have
missing_groups = set()
missing_users = set()
ulut = dict()
for m in Es.by_name('leden').get_members():
if not m.got_unix_user:
continue
ulut[m._id] = m
users[str(m.name)] = m
# Get all groups and create a look-up-table for group membership
gs = tuple(Es.groups())
mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
mlut = dict()
for g in gs:
mlut[g._id] = []
for mrel in mrels:
mlut[mrel['with']].append(mrel['who'])
# Flatten out group membership. For instance: if Giedo is in Kasco
# and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
# unix group.
# But first split the mlut graph into a group and a non-group subgraph.
mlut_g = {} # { <group> : <members that are groups> }
mlut_ng = {} # { <group> : <members that are not groups> }
for g_id in mlut:
mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
mlut_g_tc = tc(mlut_g) # transitive closure
# Generate the { <group> : <indirect non-group members> } graph
memb_graph = {}
for g in gs:
if not g.got_unix_group:
continue
memb_graph[g._id] = set(mlut_ng[g._id])
for h_id in mlut_g_tc[g._id]:
memb_graph[g._id].update(mlut_ng[h_id])
# Fill the groups variable
for g in gs:
if not g.got_unix_group:
continue
groups[str(g.name)] = set([str(ulut[c].name)
for c in memb_graph[g._id] if c in ulut])
# Now, check which users and groups actually exist in owncloud
missing_users = set(users.keys())
missing_groups = set(groups.keys())
dc = pymysql.connect(
host=creds[0],
user=creds[1],
password=creds[2],
db=creds[3],
charset='utf8'
)
try:
with dc.cursor() as c:
c.execute("SELECT gid, uid FROM oc_group_user")
for group, user in c.fetchall():
if group not in groups:
continue
if user not in users or user not in groups[group]:
todo['removeUserFromGroup'].append((user, group))
continue
if user in groups[group]:
groups[group].remove(user)
c.execute("SELECT uid FROM oc_users")
for user, in c.fetchall():
if user not in users:
logging.info("wolk: stray user %s", user)
continue
missing_users.remove(user)
c.execute("SELECT gid FROM oc_groups")
for group, in c.fetchall():
if group not in groups:
logging.info("wolk: stray group %s", user)
continue
missing_groups.remove(group)
for user in missing_users:
todo['addUser'].append((
user,
six.text_type(users[user].humanName)
))
todo['addGroup'] = list(missing_groups)
for group, missing_members in six.iteritems(groups):
for user in missing_members:
todo['addUserToGroup'].append((user, group))
finally:
dc.close()
return todo