Example #1
0
def generate_unix_map(giedo):
    ret = {'groups': {},
           'users': {}}
    dt_now = now()
    # Get all users
    ulut = dict()
    for u in Es.users():
        if not u.got_unix_user:
            continue
        ulut[u._id] = u
        ret['users'][str(u.name)] = {
                'full_name': u.full_name,
                'expire_date': DT_MIN.strftime('%Y-%m-%d')}
    member_relations_grouped = dict()
    for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
        if rel['who'] not in member_relations_grouped:
            member_relations_grouped[rel['who']] = []
        member_relations_grouped[rel['who']].append(rel)
    for user_id, relations in member_relations_grouped.items():
        latest = max(relations, key=lambda x: x['until'])
        ret['users'][str(ulut[user_id].name)]['expire_date'] \
                = latest['until'].strftime('%Y-%m-%d')

    # Get all groups and create a look-up-table for group membership
    gs = tuple(Es.groups())
    mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
                until=dt_now)
    mlut = dict()
    for g in gs:
        mlut[g._id] = []
    for mrel in mrels:
        mlut[mrel['with']].append(mrel['who'])
    # Flatten out group membership.  For instance: if Giedo is in Kasco
    # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
    # unix group.
    # But first split the mlut graph into a group and a non-group subgraph.
    mlut_g = {}     # { <group> : <members that are groups> }
    mlut_ng = {}    # { <group> : <members that are not groups> }
    for g_id in mlut:
        mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
        mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
    mlut_g_tc = tc(mlut_g)  # transitive closure
    # Generate the { <group> : <indirect non-group members> } graph
    memb_graph = {}
    for g in gs:
        if not g.got_unix_group:
            continue
        memb_graph[g._id] = set(mlut_ng[g._id])
        for h_id in mlut_g_tc[g._id]:
            memb_graph[g._id].update(mlut_ng[h_id])
    # Fill the return map
    for g in gs:
        if not g.got_unix_group:
            continue
        ret['groups'][str(g.name)] = [str(ulut[c].name)
                for c in memb_graph[g._id] if c in ulut]
    return ret
Example #2
0
def generate_unix_map(giedo):
    ret = {'groups': {},
           'users': {}}
    dt_now = now()
    # Get all users
    ulut = dict()
    for u in Es.users():
        if not u.got_unix_user:
            continue
        ulut[u._id] = u
        ret['users'][str(u.name)] = {
                'full_name': u.full_name,
                'expire_date': DT_MIN.strftime('%Y-%m-%d')}
    member_relations_grouped = dict()
    for rel in Es.query_relations(_with=Es.by_name('leden'), until=dt_now):
        if rel['who'] not in member_relations_grouped:
            member_relations_grouped[rel['who']] = []
        member_relations_grouped[rel['who']].append(rel)
    for user_id, relations in member_relations_grouped.items():
        latest = max(relations, key=lambda x: x['until'])
        ret['users'][str(ulut[user_id].name)]['expire_date'] \
                = latest['until'].strftime('%Y-%m-%d')

    # Get all groups and create a look-up-table for group membership
    gs = tuple(Es.groups())
    mrels = Es.query_relations(how=None, _with=gs, _from=dt_now,
                until=dt_now)
    mlut = dict()
    for g in gs:
        mlut[g._id] = []
    for mrel in mrels:
        mlut[mrel['with']].append(mrel['who'])
    # Flatten out group membership.  For instance: if Giedo is in Kasco
    # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
    # unix group.
    # But first split the mlut graph into a group and a non-group subgraph.
    mlut_g = {}     # { <group> : <members that are groups> }
    mlut_ng = {}    # { <group> : <members that are not groups> }
    for g_id in mlut:
        mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
        mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
    mlut_g_tc = tc(mlut_g)  # transitive closure
    # Generate the { <group> : <indirect non-group members> } graph
    memb_graph = {}
    for g in gs:
        if not g.got_unix_group:
            continue
        memb_graph[g._id] = set(mlut_ng[g._id])
        for h_id in mlut_g_tc[g._id]:
            memb_graph[g._id].update(mlut_ng[h_id])
    # Fill the return map
    for g in gs:
        if not g.got_unix_group:
            continue
        ret['groups'][str(g.name)] = [str(ulut[c].name)
                for c in memb_graph[g._id] if c in ulut]
    return ret
Example #3
0
def main():
    now = Es.now()
    for n, r in enumerate(sorted(Es.query_relations(
            _with=Es.id_by_name('leden'), _from=now, until=now,
            deref_who=True),
                key=lambda r: r['from'])): 
        print n+1, r['who'].humanName
Example #4
0
def generate_postfix_map(giedo):
    tbl = dict()  # the virtual map
    non_mailman_groups = {}
    dt_now = now()
    id2email = {}
    # handle the straightforward cases
    for e in Es.all():
        if e.canonical_email is None:
            continue
        id2email[e._id] = e.canonical_email
        for nm in e.other_names:
            tbl["%s@%s" % (nm, settings.MAILDOMAIN)] = (e.canonical_email,)
        if e.type == 'user':
            tbl[e.canonical_email] = (e.email,)
        elif e.type == 'group':
            if e.got_mailman_list and e.name:
                tbl[e.canonical_email] = ('%s@%s' % (
                    str(e.name), settings.LISTS_MAILDOMAIN),)
            else:
                tbl[e.canonical_email] = []
                non_mailman_groups[e._id] = e
        else:
            logging.warn("postfix: unhandled type: %s" % e.type)
        id_email = "%s@%s" % (e.id, settings.MAILDOMAIN)
        if id_email not in tbl:
            tbl[id_email] = (e.canonical_email,)
    # handle the non-mailman groups
    for rel in Es.query_relations(_with=list(non_mailman_groups),
                                  _from=dt_now, until=dt_now, how=None):
        e = non_mailman_groups[rel['with']]
        email = id2email.get(rel['who'])
        if email is not None:
            tbl[e.canonical_email].append(email)
    return tbl
Example #5
0
def relation_begin(request):
    # TODO We should use Django forms, or better: use sweet Ajax
    d = {}
    for t in ('who', 'with', 'how'):
        if t not in request.POST:
            raise ValueError, "Missing attr %s" % t
        if t == 'how' and request.POST[t] == 'null':
            d[t] = None
        else:
            d[t] = _id(request.POST[t])
    if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
                                                                d['how']):
        raise PermissionDenied
    # Check whether such a relation already exists
    dt = now()
    ok = False
    try:
        next(Es.query_relations(who=d['who'], _with=d['with'],
            how=d['how'], _from=dt, until=DT_MAX))
    except StopIteration:
        ok = True
    if not ok:
        raise ValueError, "This relation already exists"
    # Add the relation!
    Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)
    giedo.sync()
    return redirect_to_referer(request)
Example #6
0
def main2():
    rels = list(Es.query_relations(-1,
                                   Es.by_name('comms').get_bearers(),
                                   None,
                                   now(),
                                   deref_who=True,
                                   deref_with=True))
    lut = {}
    for rel in rels:
        if rel['from'] is None:
            rel['from'] = Es.DT_MIN
        if not rel['with'] in lut:
            lut[rel['with']] = {}
        v = (now() - rel['from']).days / 365.0
        if rel['who'] not in lut[rel['with']] or \
                lut[rel['with']][rel['who']] > v:
            lut[rel['with']][rel['who']] = v

    pairs = lut.items()
    for comm, members in pairs:
        print(six.text_type(comm.humanName))
        mpairs = members.items()
        mpairs.sort(key=lambda x: x[1])
        for member, time in mpairs:
            print(' %-20s%.2f' % (member.name, time))
Example #7
0
def main2():
    rels = list(
        Es.query_relations(-1,
                           Es.by_name('comms').get_bearers(),
                           None,
                           now(),
                           deref_who=True,
                           deref_with=True))
    lut = {}
    for rel in rels:
        if rel['from'] is None:
            rel['from'] = Es.DT_MIN
        if not rel['with'] in lut:
            lut[rel['with']] = {}
        v = (now() - rel['from']).days / 365.0
        if rel['who'] not in lut[rel['with']] or \
                lut[rel['with']][rel['who']] > v:
            lut[rel['with']][rel['who']] = v

    pairs = lut.items()
    for comm, members in pairs:
        print(six.text_type(comm.humanName))
        mpairs = members.items()
        mpairs.sort(key=lambda x: x[1])
        for member, time in mpairs:
            print(' %-20s%.2f' % (member.name, time))
Example #8
0
def main():
    now = Es.now()
    member_age = {}
    for rel in Es.query_relations(-1,
                                  Es.by_name('leden'),
                                  None,
                                  None,
                                  deref_who=True):
        if not rel['from']:
            rel['from'] = Es.DT_MIN
        if rel['who'] not in member_age:
            member_age[rel['who']] = Es.DT_MAX
        member_age[rel['who']] = rel['from']
    l = []
    for m, age in member_age.iteritems():
        if not m.dateOfBirth:
            continue
        print(age - m.dateOfBirth).days / 365.242, unicode(m.name)
        l.append((age - m.dateOfBirth).days / 365.242)
    print 'avg', sum(l) / len(l)
    print 'med', sorted(l)[len(l) / 2]
    print '1st', sorted(l)[len(l) / 4 * 2]
    print '3rd', sorted(l)[len(l) / 4 * 3]
    print 'min', min(l)
    print 'max', max(l)
Example #9
0
def generate_postfix_map(giedo):
        tbl = dict() # the virtual map
        non_mailman_groups = {}
        dt_now = now()
        id2email = {}
        # handle the straightforward cases
        for e in Es.all():
                if e.canonical_email is None or e.name is None:
                        continue
                id2email[e._id] = e.canonical_email
                for nm in e.other_names:
                        tbl["%s@%s" % (nm, MAILDOMAIN)] = (e.canonical_email,)
                if e.type == 'user':
                        tbl[e.canonical_email] = (e.primary_email,)
                elif e.type == 'group':
                        if e.got_mailman_list:
                                tbl[e.canonical_email] = ('%s@%s' % (
                                        str(e.name), LISTS_MAILDOMAIN),)
                        else:
                                tbl[e.canonical_email] = []
                                non_mailman_groups[e._id] = e
                else:
                        logging.warn("postfix: unhandled type: %s" % e.type)
        # handle the non-mailman groups
        for rel in Es.query_relations(_with=non_mailman_groups.keys(),
                        _from=dt_now, until=dt_now, how=None):
                e = non_mailman_groups[rel['with']]
                email = id2email.get(rel['who'])
                if email is not None:
                        tbl[e.canonical_email].append(email)
        return tbl
Example #10
0
def relation_begin(request):
    # TODO We should use Django forms, or better: use sweet Ajax
    d = {}
    for t in ("who", "with", "how"):
        if t not in request.POST:
            raise ValueError, "Missing attr %s" % t
        if t == "how" and (not request.POST[t] or request.POST[t] == "null"):
            d[t] = None
        else:
            d[t] = _id(request.POST[t])
    if not Es.user_may_begin_relation(request.user, d["who"], d["with"], d["how"]):
        raise PermissionDenied

    # Check whether such a relation already exists
    dt = now()
    ok = False
    try:
        next(Es.query_relations(who=d["who"], _with=d["with"], how=d["how"], _from=dt, until=DT_MAX))
    except StopIteration:
        ok = True
    if not ok:
        raise ValueError, _("Deze relatie bestaat al")

    # Add the relation!
    relation_id = Es.add_relation(d["who"], d["with"], d["how"], dt, DT_MAX)

    # Notify informacie
    # TODO (rik) leave out 'als lid'
    Es.notify_informacie("relation_begin", request.user, relation=relation_id)

    giedo.sync_async(request)
    return redirect_to_referer(request)
Example #11
0
def generate_postfix_slm_map(giedo):
    # We generate the postfix "sender_login_maps".
    # This is used to decide by postfix whether a given user is allowed to
    # send e-mail as if it was coming from a particular e-mail address.
    # It is a dictionary { <email address> : [ <allowed user>, ... ] }
    tbl = dict()
    dt_now = now()
    # Get all users
    ulut = dict()
    # We only allow members to send e-mail
    for u in Es.by_name('leden').get_members():
        ulut[u._id] = u
        for name in u.names:
            if str(name) not in tbl:
                tbl[str(name)] = set()
            tbl[str(name)].add(str(u.name))
    # There are two kind of groups: groups whose members are allowed to
    # send e-mail as if coming from the group itself and those where this
    # is not allowed.  For convenience, lets call the first kind the
    # impersonatable groups.
    # Get all impersonatable groups and create a look-up-table for
    # group membership
    gs = list()
    for g in Es.groups():
        # TODO add a tag to force a group either impersonatable or not
        if not g.got_mailman_list:
            gs.append(g)
    mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
    mlut = dict()
    for g in gs:
        mlut[g._id] = []
    for mrel in mrels:
        mlut[mrel['with']].append(mrel['who'])
    # Flatten out group membership.  For instance: if Giedo is in Kasco
    # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
    # unix group.
    # But first split the mlut graph into a impersonatable group
    # and a non-group subgraph.
    mlut_g = {}     # { <group> : <members that are impersonatable groups> }
    mlut_u = {}    # { <group> : <members that are users> }
    for g_id in mlut:
        mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
        mlut_u[g_id] = [c for c in mlut[g_id] if c in ulut]
    mlut_g_tc = tc(mlut_g)  # transitive closure
    for g in gs:
        to_consider = tuple(mlut_g_tc[g._id]) + (g._id,)
        for sg_id in to_consider:
            for u_id in mlut_u[sg_id]:
                for name in g.names:
                    if str(name) not in tbl:
                        tbl[str(name)] = set()
                    tbl[str(name)].add(str(ulut[u_id].name))
    # Clean up tbl to return.
    ret = {}
    for name, users in six.iteritems(tbl):
        if not users:
            continue
        ret["%s@%s" % (name, settings.MAILDOMAIN)] = tuple(users)
    return ret
Example #12
0
def main():
    now = Es.now()
    for n, r in enumerate(
            sorted(Es.query_relations(_with=Es.id_by_name('leden'),
                                      _from=now,
                                      until=now,
                                      deref_who=True),
                   key=lambda r: r['from'])):
        print(n + 1, r['who'].humanName)
Example #13
0
def generate_mailman_changes(giedo):
    todo = {'create': [], 'add': {}, 'remove': {}}
    # TODO mm_groups and mm_rels
    # Get the groups that need a mailing list and the members of those
    # groups.
    dt_now = now()
    mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
    mm_rels = Es.query_relations(_with=mm_groups,
                                 how=None,
                                 _from=dt_now,
                                 until=dt_now,
                                 deref_who=True)
    # Get the current mailing list membersip
    ml_membership = giedo.hans.send({'type': 'maillist-get-membership'})

    # membership_to_check will contain the current membership of the
    # mailinglists for which there are groups.  We will remove users
    # from membership_to_check if we see these users are in the groups.
    # In the end membership_to_check contains the stray users.
    membership_to_check = {}
    gid2name = dict()
    # Find the current members of the mailing lists and find which
    # mailing lists are missing.
    for g in mm_groups:
        name = str(g.name).encode()
        gid2name[g._id] = name
        if name in ml_membership:
            membership_to_check[name] = set(ml_membership[name])
        else:
            todo['create'].append((name, six.text_type(g.humanName)))
            membership_to_check[name] = set()

    # Check which memberships are missing in the current mailing lists
    for rel in mm_rels:
        em = rel['who'].canonical_email.encode()
        gname = gid2name[rel['with']]
        if em not in membership_to_check[gname]:
            if gname not in todo['add']:
                todo['add'][gname] = []
            todo['add'][gname].append(em)
        else:
            membership_to_check[gname].remove(em)

    # Check which memberships are superfluous in the current mailing lists
    for n in ml_membership:
        if n not in membership_to_check:
            logging.warning("Unaccounted e-maillist %s" % n)
            continue
        for em in membership_to_check[n]:
            if n not in todo['remove']:
                todo['remove'][n] = []
            todo['remove'][n].append(em)
    return todo
Example #14
0
def _brand_detail(request, brand):
    ctx = _entity_detail(request, brand)
    ctx['rels'] = sorted(
        Es.query_relations(how=brand, deref_who=True, deref_with=True),
        key=lambda x: (Es.DT_MIN - Es.relation_until(
            x), Es.entity_humanName(x['with']), Es.entity_humanName(x['who'])))
    for r in ctx['rels']:
        r['id'] = r['_id']
        r['until_year'] = (None if r['until'] is None else Es.date_to_year(
            r['until']))
        r['virtual'] = Es.relation_is_virtual(r)
    return render(request, 'leden/brand_detail.html', ctx)
Example #15
0
def _brand_detail(request, brand):
    ctx = _entity_detail(request, brand)
    ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
                                            deref_with=True),
                         key=lambda x: (Es.DT_MIN - Es.relation_until(x),
                                        Es.entity_humanName(x['with']),
                                        Es.entity_humanName(x['who'])))
    for r in ctx['rels']:
        r['id'] = r['_id']
        r['until_year'] = (None if r['until'] is None else
                           Es.date_to_year(r['until']))
        r['virtual'] = Es.relation_is_virtual(r)
    return render(request, 'leden/brand_detail.html', ctx)
Example #16
0
def qrel(who=-1, _with=-1, how=-1, _from=None, until=None):
    """ Queries relations """
    if who not in (None, -1):
        who = Es.id_by_name(who)
    if _with not in (None, -1):
        _with = Es.id_by_name(_with)
    if how not in (None, -1):
        how = Es.ecol.find_one({'sofa_suffix': how})['_id']
    if _from not in (None, -1):
        _from = str_to_date(_from)
    if until not in (None, -1):
        until = str_to_date(until)
    return list(Es.query_relations(who, _with, how, _from, until, True,
                                   True, True))
Example #17
0
def qrel(who=-1, _with=-1, how=-1, _from=None, until=None):
    """ Queries relations """
    if who not in (None, -1):
        who = Es.id_by_name(who)
    if _with not in (None, -1):
        _with = Es.id_by_name(_with)
    if how not in (None, -1):
        how = Es.ecol.find_one({'sofa_suffix': how})['_id']
    if _from not in (None, -1):
        _from = str_to_date(_from)
    if until not in (None, -1):
        until = str_to_date(until)
    return list(
        Es.query_relations(who, _with, how, _from, until, True, True, True))
Example #18
0
def _group_detail(request, group):
    ctx = _entity_detail(request, group)
    isFreeToJoin = group.has_tag(Es.id_by_name("!free-to-join", True))
    rel_id = None
    if isFreeToJoin:
        dt = now()
        rel = list(Es.query_relations(request.user, group, None, dt, dt, False, False, False))
        assert len(rel) <= 1
        for r in rel:
            rel_id = r["_id"]
    ctx.update(
        {"isFreeToJoin": group.has_tag(Es.by_name("!free-to-join")), "request": request, "relation_with_group": rel_id}
    )
    return render_to_response("leden/group_detail.html", ctx, context_instance=RequestContext(request))
Example #19
0
 def _sync_villanet(self):
     ret = self.villanet_request({'action': 'listUsers'})
     if not ret[0]:
         return
     ret = json.loads(ret[1])
     users = dict()
     ulut = dict()
     for u in Es.users():
         ulut[u._id] = str(u.name)
     member_relations_grouped = dict()
     for rel in Es.query_relations(_with=Es.by_name('leden'), until=now()):
         if rel['who'] not in member_relations_grouped:
             member_relations_grouped[rel['who']] = []
         member_relations_grouped[rel['who']].append(rel)
     for user_id, relations in member_relations_grouped.items():
         latest = max(relations, key=lambda x: x['until'])
         users[ulut[user_id]] = latest['until'].strftime('%Y-%m-%d')
     vn = set(ret.keys())
     kn = set(users.keys())
     dt_max = settings.DT_MAX.strftime('%Y-%m-%d')
     for name in kn - vn:
         data = {
             'username': name,
             'password': self.villanet_encrypt_password(pseudo_randstr(16)),
         }
         if users[name] != dt_max:
             data['till'] = users[name]
         pc = Es.PushChange({
             'system': 'villanet',
             'action': 'addUser',
             'data': data
         })
         pc.save()
     for name in vn - kn:
         logging.info("Stray user %s" % name)
     for name in vn & kn:
         remote = (ret[name]['till'][:10]
                   if ret[name]['till'] is not None else '')
         local = users[name] if users[name] != dt_max else ''
         if remote != local:
             pc = Es.PushChange({
                 'system': 'villanet',
                 'action': 'changeUser',
                 'data': {
                     'username': name,
                     'till': local
                 }
             })
             pc.save()
     self.push_changes_event.set()
Example #20
0
def generate_mailman_changes(giedo):
    todo = {'create': [], 'add': {}, 'remove': {}}
    # TODO mm_groups and mm_rels
    # Get the groups that need a mailing list and the members of those
    # groups.
    dt_now = now()
    mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
    mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
                                 until=dt_now, deref_who=True)
    # Get the current mailing list membersip
    ml_membership = giedo.hans.send({'type': 'maillist-get-membership'})

    # membership_to_check will contain the current membership of the
    # mailinglists for which there are groups.  We will remove users
    # from membership_to_check if we see these users are in the groups.
    # In the end membership_to_check contains the stray users.
    membership_to_check = {}
    gid2name = dict()
    # Find the current members of the mailing lists and find which
    # mailing lists are missing.
    for g in mm_groups:
        name = str(g.name).encode()
        gid2name[g._id] = name
        if name in ml_membership:
            membership_to_check[name] = set(ml_membership[name])
        else:
            todo['create'].append((name, six.text_type(g.humanName)))
            membership_to_check[name] = set()

    # Check which memberships are missing in the current mailing lists
    for rel in mm_rels:
        em = rel['who'].canonical_email.encode()
        gname = gid2name[rel['with']]
        if em not in membership_to_check[gname]:
            if gname not in todo['add']:
                todo['add'][gname] = []
            todo['add'][gname].append(em)
        else:
            membership_to_check[gname].remove(em)

    # Check which memberships are superfluous in the current mailing lists
    for n in ml_membership:
        if n not in membership_to_check:
            logging.warning("Unaccounted e-maillist %s" % n)
            continue
        for em in membership_to_check[n]:
            if n not in todo['remove']:
                todo['remove'][n] = []
            todo['remove'][n].append(em)
    return todo
Example #21
0
def _group_detail(request, group):
    ctx = _entity_detail(request, group)
    isFreeToJoin = group.has_tag(Es.id_by_name('!free-to-join', True))
    rel_id = None
    if isFreeToJoin:
        dt = now()
        rel = list(Es.query_relations(request.user, group, None,
                                      dt, dt, False, False, False))
        assert len(rel) <= 1
        for r in rel:
            rel_id = r['_id']
    ctx.update({'isFreeToJoin': group.has_tag(Es.by_name('!free-to-join')),
                'request': request,
                'relation_with_group': rel_id})
    return render(request, 'leden/group_detail.html', ctx)
Example #22
0
def generate_mailman_changes(hans):
    todo = hans_pb2.ApplyChangesReq()
    # TODO mm_groups and mm_rels
    # Get the groups that need a mailing list and the members of those
    # groups.
    dt_now = now()
    mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
    mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
                                 until=dt_now, deref_who=True)
    # Get the current mailing list membersip
    ml_membership = hans.GetMembership(hans_pb2.GetMembershipReq()).membership

    # membership_to_check will contain the current membership of the
    # mailinglists for which there are groups.  We will remove users
    # from membership_to_check if we see these users are in the groups.
    # In the end membership_to_check contains the stray users.
    membership_to_check = {}
    gid2name = dict()
    # Find the current members of the mailing lists and find which
    # mailing lists are missing.
    for g in mm_groups:
        name = str(g.name).encode()
        gid2name[g._id] = name
        if name in ml_membership:
            membership_to_check[name] = set(ml_membership[name].emails)
        else:
            todo.create.append(hans_pb2.ListCreateReq(
                name=name,
                humanName=six.text_type(g.humanName)))
            membership_to_check[name] = set()

    # Check which memberships are missing in the current mailing lists
    for rel in mm_rels:
        em = rel['who'].canonical_email.encode()
        gname = gid2name[rel['with']]
        if em not in membership_to_check[gname]:
            todo.add[gname].emails.append(em)
        else:
            membership_to_check[gname].remove(em)

    # Check which memberships are superfluous in the current mailing lists
    for n in ml_membership:
        if n not in membership_to_check:
            logging.warning("Unaccounted e-maillist %s" % n)
            continue
        for em in membership_to_check[n]:
            todo.remove[n].emails.append(em)
    return todo
Example #23
0
 def _sync_villanet(self):
     if not settings.VILLANET_SECRET_API_KEY:
         logging.warn("VILLANET_SECRET_API_KEY not set")
         return
     ret = self.villanet_request({'action': 'listUsers'})
     if not ret[0]:
         return
     ret = json.loads(ret[1])
     users = dict()
     ulut = dict()
     for u in Es.users():
         ulut[u._id] = str(u.name)
     member_relations_grouped = dict()
     for rel in Es.query_relations(_with=Es.by_name('leden'), until=now()):
         if rel['who'] not in member_relations_grouped:
             member_relations_grouped[rel['who']] = []
         member_relations_grouped[rel['who']].append(rel)
     for user_id, relations in member_relations_grouped.items():
         latest = max(relations, key=lambda x: x['until'])
         users[ulut[user_id]] = latest['until'].strftime('%Y-%m-%d')
     vn = set(ret.keys())
     kn = set(users.keys())
     dt_max = settings.DT_MAX.strftime('%Y-%m-%d')
     for name in kn - vn:
         data = {
                 'username': name,
                 'password': self.villanet_encrypt_password(
                     pseudo_randstr(16)),
             }
         if users[name] != dt_max:
             data['till'] = users[name]
         pc = Es.PushChange({'system': 'villanet', 'action': 'addUser',
             'data': data})
         pc.save()
     for name in vn - kn:
         logging.info("Stray user %s" % name)
     for name in vn & kn:
         remote = (ret[name]['till'][:10] if ret[name]['till'] is not None
                 else '')
         local = users[name] if users[name] != dt_max else ''
         if remote != local:
             pc = Es.PushChange({'system': 'villanet',
                 'action': 'changeUser', 'data': {
                     'username': name,
                     'till': local
                     }})
             pc.save()
     self.push_changes_event.set()
Example #24
0
def _group_detail(request, group):
    ctx = _entity_detail(request, group)
    isFreeToJoin = group.has_tag(Es.id_by_name('!free-to-join', True))
    rel_id = None
    if isFreeToJoin:
        dt = now()
        rel = list(Es.query_relations(request.user, group, None,
                                        dt, dt, False, False, False))
        assert len(rel) <= 1
        for r in rel:
            rel_id = r['_id']
    ctx.update({'isFreeToJoin': group.has_tag(Es.by_name('!free-to-join')),
        'request': request,
        'relation_with_group': rel_id})
    return render_to_response('leden/group_detail.html', ctx,
            context_instance=RequestContext(request))
Example #25
0
def generate_mailman_changes(giedo):
    todo = {'create': [], 'add': {}, 'remove': {}}
    # TODO mm_groups and mm_rels
    # Get the groups that need a mailing list and the members of those
    # groups.
    dt_now = now()
    mm_groups = [g for g in Es.groups() if g.got_mailman_list and g.name]
    mm_rels = Es.query_relations(_with=mm_groups, how=None, _from=dt_now,
            until=dt_now, deref_who=True)
    # TODO do we want to cache these?
    # Get the current mailing lists
    ml_names = frozenset(Mailman.Utils.list_names())
    ml_members = dict()
    gid2name = dict()
    # Find the current members of the mailing lists and find which
    # mailing lists are missing.
    for g in mm_groups:
        gid2name[g._id] = str(g.name)
        if not str(g.name) in ml_names:
            todo['create'].append((str(g.name),
                    unicode(g.humanName)))
            c_ms = set([])
        else:
            c_ms = set([x[0] for x in
                Mailman.MailList.MailList(str(g.name),
                    lock=False).members.iteritems()])
        ml_members[str(g.name)] = c_ms
    # Check which memberships are missing in the current mailing lists
    for rel in mm_rels:
        em = rel['who'].canonical_email
        gname = gid2name[rel['with']]
        if not em in ml_members[gname]:
            if not gname in todo['add']:
                todo['add'][gname] = []
            todo['add'][gname].append(em)
        else:
            ml_members[gname].remove(em)
    # Check which memberships are superfluous in the current mailing lists
    for n in ml_names:
        if not n in ml_members:
            logging.warning("Unaccounted e-maillist %s" % n)
            continue
        for em in ml_members[n]:
            if not n in todo['remove']:
                todo['remove'][n] = []
            todo['remove'][n].append(em)
    return todo
Example #26
0
def main3():
    member_age = {}
    for rel in Es.query_relations(-1, Es.by_name('leden'), None,
                                  None, deref_who=True):
        if rel['who'] not in member_age:
            member_age[rel['who']] = 0
        member_age[rel['who']] = max(member_age[rel['who']],
                                     (now() - rel['from']).days / 365.0)

    # for comm in Es.by_name('comms').get_bearers():
    for comm in [Es.by_name('draai')]:
        print(six.text_type(comm.humanName))
        members = [(m, member_age.get(m)) for m in comm.get_members()]
        members.sort(key=lambda x: x[1])
        for member in members:
            print(" %-20s%.2f" % (six.text_type(member[0].name),
                                  member[1] if member[1] else -1))
def main():
    year_overrides = dict()
    had = set()
    for t in Es.bearers_by_tag_id(
            Es.id_by_name('!year-overrides'),
            _as=Es.Tag):
        year_overrides[(t._data['year-override']['type'],
                        t._data['year-override']['year'])] = t._data['_id']
    for rel in sorted(Es.query_relations(-1, Es.id_by_name('leden'), None,
                                         None, None, True, False, False),
                      key=lambda x: x['from']):
        name = str(rel['who'].name)
        if name in had:
            continue
        if ('temp' not in rel['who']._data or
                'joined' not in rel['who']._data['temp']):
            had.add(name)
            continue
        joined = datetime.datetime.strptime(
            rel['who']._data['temp']['joined'], '%Y-%m-%d')
        if joined == rel['from']:
            had.add(name)
            continue
        joined_yr = Es.date_to_year(joined)
        from_yr = Es.date_to_year(rel['from'])
        if joined_yr == from_yr or joined_yr == 0:
            print(name, rel['from'].date(), ' -> ',
                  joined.date())
            rrel = Es.rcol.find({'_id': rel['_id']})[0]
            rrel['from'] = joined
            Es.rcol.save(rrel)
            had.add(name)
            continue
        if joined_yr == from_yr - 1:
            print(name, rel['from'].date(), ' -> ',
                  joined.date())
            rrel = Es.rcol.find({'_id': rel['_id']})[0]
            rrel['from'] = joined
            if 'tags' not in rrel:
                rrel['tags'] = []
            if not year_overrides[(False, joined_yr)] in rrel['tags']:
                rrel['tags'].append(year_overrides[(False, joined_yr)])
            Es.rcol.save(rrel)
            had.add(name)
            continue
        print('MANUAL REVIEW: ', name, joined.date(), rel['from'].date())
Example #28
0
def main():
    year_overrides = dict()
    had = set()
    for t in Es.bearers_by_tag_id(Es.id_by_name('!year-overrides'),
                                  _as=Es.Tag):
        year_overrides[(t._data['year-override']['type'],
                        t._data['year-override']['year'])] = t._data['_id']
    for rel in sorted(Es.query_relations(-1, Es.id_by_name('leden'), None,
                                         None, None, True, False, False),
                      cmp=lambda x, y: cmp(x['from'], y['from'])):
        name = str(rel['who'].name)
        if name in had:
            continue
        if ('temp' not in rel['who']._data
                or 'joined' not in rel['who']._data['temp']):
            had.add(name)
            continue
        joined = datetime.datetime.strptime(rel['who']._data['temp']['joined'],
                                            '%Y-%m-%d')
        if joined == rel['from']:
            had.add(name)
            continue
        joined_yr = Es.date_to_year(joined)
        from_yr = Es.date_to_year(rel['from'])
        if joined_yr == from_yr or joined_yr == 0:
            print name, rel['from'].date(), ' -> ', \
                                joined.date()
            rrel = Es.rcol.find({'_id': rel['_id']})[0]
            rrel['from'] = joined
            Es.rcol.save(rrel)
            had.add(name)
            continue
        if joined_yr == from_yr - 1:
            print name, rel['from'].date(), ' -> ', \
                                joined.date()
            rrel = Es.rcol.find({'_id': rel['_id']})[0]
            rrel['from'] = joined
            if not 'tags' in rrel:
                rrel['tags'] = []
            if not year_overrides[(False, joined_yr)] in rrel['tags']:
                rrel['tags'].append(year_overrides[(False, joined_yr)])
            Es.rcol.save(rrel)
            had.add(name)
            continue
        print 'MANUAL REVIEW: ', name, \
                    joined.date(), rel['from'].date()
Example #29
0
def main3():
    member_age = {}
    for rel in Es.query_relations(-1,
                                  Es.by_name('leden'),
                                  None,
                                  None,
                                  deref_who=True):
        if rel['who'] not in member_age:
            member_age[rel['who']] = 0
        member_age[rel['who']] = max(member_age[rel['who']],
                                     (now() - rel['from']).days / 365.0)

    # for comm in Es.by_name('comms').get_bearers():
    for comm in [Es.by_name('draai')]:
        print(six.text_type(comm.humanName))
        members = [(m, member_age.get(m)) for m in comm.get_members()]
        members.sort(key=lambda x: x[1])
        for member in members:
            print(" %-20s%.2f" % (six.text_type(
                member[0].name), member[1] if member[1] else -1))
Example #30
0
def _brand_detail(request, brand):
    ctx = _entity_detail(request, brand)

    def _cmp(x, y):
        r = Es.relation_cmp_until(y, x)
        if r:
            return r
        r = cmp(unicode(x["with"].humanName), unicode(y["with"].humanName))
        if r:
            return r
        r = cmp(unicode(x["who"].humanName), unicode(y["who"].humanName))
        if r:
            return r
        return Es.relation_cmp_from(x, y)

    ctx["rels"] = sorted(Es.query_relations(how=brand, deref_who=True, deref_with=True), cmp=_cmp)
    for r in ctx["rels"]:
        r["id"] = r["_id"]
        r["until_year"] = None if r["until"] is None else Es.date_to_year(r["until"])
        r["virtual"] = Es.relation_is_virtual(r)
    return render_to_response("leden/brand_detail.html", ctx, context_instance=RequestContext(request))
Example #31
0
def _brand_detail(request, brand):
    ctx = _entity_detail(request, brand)
    def _cmp(x,y):
        r = Es.relation_cmp_until(y,x)
        if r: return r
        r = cmp(unicode(x['with'].humanName),
                unicode(y['with'].humanName))
        if r: return r
        r = cmp(unicode(x['who'].humanName),
                unicode(y['who'].humanName))
        if r: return r
        return Es.relation_cmp_from(x,y)
    ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
                deref_with=True), cmp=_cmp)
    for r in ctx['rels']:
        r['id'] = r['_id']
        r['until_year'] = (None if r['until'] is None else
                    Es.date_to_year(r['until']))
        r['virtual'] = Es.relation_is_virtual(r)
    return render_to_response('leden/brand_detail.html', ctx,
            context_instance=RequestContext(request))
Example #32
0
def _brand_detail(request, brand):
    ctx = _entity_detail(request, brand)
    def _cmp(x,y):
        r = Es.relation_cmp_until(y,x)
        if r: return r
        r = cmp(unicode(x['with'].humanName),
                unicode(y['with'].humanName))
        if r: return r
        r = cmp(unicode(x['who'].humanName),
                unicode(y['who'].humanName))
        if r: return r
        return Es.relation_cmp_from(x,y)
    ctx['rels'] = sorted(Es.query_relations(how=brand, deref_who=True,
                deref_with=True), cmp=_cmp)
    for r in ctx['rels']:
        r['id'] = r['_id']
        r['until_year'] = (None if r['until'] is None else
                    Es.date_to_year(r['until']))
        r['virtual'] = Es.relation_is_virtual(r)
    return render_to_response('leden/brand_detail.html', ctx,
            context_instance=RequestContext(request))
Example #33
0
def main():
    member_age = {}
    for rel in Es.query_relations(-1, Es.by_name('leden'), None,
                                  None, deref_who=True):
        if not rel['from']:
            rel['from'] = Es.DT_MIN
        if rel['who'] not in member_age:
            member_age[rel['who']] = Es.DT_MAX
        member_age[rel['who']] = rel['from']
    le = []
    for m, age in six.iteritems(member_age):
        if not m.dateOfBirth:
            continue
        print((age - m.dateOfBirth).days / 365.242, six.text_type(m.name))
        le.append((age - m.dateOfBirth).days / 365.242)
    print('avg', sum(le) / len(le))
    print('med', sorted(le)[len(le) / 2])
    print('1st', sorted(le)[len(le) / 4 * 2])
    print('3rd', sorted(le)[len(le) / 4 * 3])
    print('min', min(le))
    print('max', max(le))
Example #34
0
def _generate_member_count():
    events = []
    for rel in Es.query_relations(_with=Es.id_by_name('leden'), how=None):
        events.append((max(rel['from'], Es.DT_MIN), True))
        if rel['until'] != Es.DT_MAX:
            events.append((rel['until'], False))
    N = 0
    old_days = -1
    old_N = None
    ret = []
    for when, what in sorted(events, key=lambda x: x[0]):
        N += 1 if what else -1
        days = (when - Es.DT_MIN).days
        if old_days != days:
            if old_N:
                ret.append([old_days, old_N])
            old_days = days
            old_N = N
    ret.append([days, N])
    ret = [(1 + days2 / 365.242, N2) for days2, N2 in ret]
    return ret
Example #35
0
def _generate_member_count():
    events = []
    for rel in Es.query_relations(_with=Es.id_by_name('leden'), how=None):
        events.append((max(rel['from'], Es.DT_MIN), True))
        if rel['until'] != Es.DT_MAX:
            events.append((rel['until'], False))
    N = 0
    old_days = -1
    old_N = None
    ret = []
    for when, what in sorted(events, key=lambda x: x[0]):
        N += 1 if what else -1
        days = (when - Es.DT_MIN).days
        if old_days != days:
            if old_N:
                ret.append([old_days, old_N])
            old_days = days
            old_N = N
    ret.append([days, N])
    ret = [(1 + days / 365.242, N) for days, N in ret]
    return ret
Example #36
0
def relation_begin(request):
    # TODO We should use Django forms, or better: use sweet Ajax
    d = {}
    for t in ('who', 'with', 'how'):
        if t not in request.POST:
            raise ValueError, "Missing attr %s" % t
        if t == 'how' and (not request.POST[t] or
                               request.POST[t] == 'null'):
            d[t] = None
        else:
            d[t] = _id(request.POST[t])
    if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
                                                                d['how']):
        raise PermissionDenied
    # Check whether such a relation already exists
    dt = now()
    ok = False
    try:
        next(Es.query_relations(who=d['who'], _with=d['with'],
            how=d['how'], _from=dt, until=DT_MAX))
    except StopIteration:
        ok = True
    if not ok:
        raise ValueError, "This relation already exists"
    # Add the relation!
    Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)
    # Notify informacie
    if request.user._id == d['who']:
        Es.notify_informacie("%s heeft zich ingeschreven als %s %s" % (
                            request.user.full_name,
                            Es.by_id(d['how']).humanName if d['how'] else 'lid',
                            Es.by_id(d['with']).humanName.genitive))
    else:
        # TODO (rik) leave out 'als lid'
        Es.notify_informacie("%s is nu %s %s" % (
                            Es.by_id(d['who']).humanName,
                            Es.by_id(d['how']).humanName if d['how'] else 'lid',
                            Es.by_id(d['with']).humanName.genitive))
    giedo.sync_async(request)
    return redirect_to_referer(request)
Example #37
0
def relation_begin(request):
    # TODO We should use Django forms, or better: use sweet Ajax
    d = {}
    for t in ('who', 'with', 'how'):
        if t not in request.POST:
            raise ValueError("Missing attr %s" % t)
        if t == 'how' and (not request.POST[t] or request.POST[t] == 'null'):
            d[t] = None
        else:
            d[t] = _id(request.POST[t])
    if not Es.user_may_begin_relation(request.user, d['who'], d['with'],
                                      d['how']):
        raise PermissionDenied

    # Check whether such a relation already exists
    dt = now()
    ok = False
    try:
        next(
            Es.query_relations(who=d['who'],
                               _with=d['with'],
                               how=d['how'],
                               _from=dt,
                               until=DT_MAX))
    except StopIteration:
        ok = True
    if not ok:
        messages.info(request, _("Deze relatie bestaat al"))
        return redirect_to_referer(request)

    # Add the relation!
    relation_id = Es.add_relation(d['who'], d['with'], d['how'], dt, DT_MAX)

    # Notify informacie
    # TODO (rik) leave out 'als lid'
    Es.notify_informacie('relation_begin', request.user, relation=relation_id)

    giedo.sync_async(request)
    return redirect_to_referer(request)
Example #38
0
def main():
    now = Es.now()
    member_age = {}
    for rel in Es.query_relations(-1, Es.by_name('leden'), None,
                None, deref_who=True):
        if not rel['from']:
            rel['from'] = Es.DT_MIN
        if rel['who'] not in member_age:
            member_age[rel['who']] = Es.DT_MAX
        member_age[rel['who']] = rel['from']
    l = []
    for m, age in member_age.iteritems():
        if not m.dateOfBirth:
            continue
        print (age - m.dateOfBirth).days / 365.242, unicode(m.name)
        l.append( (age - m.dateOfBirth).days / 365.242)
    print 'avg', sum(l)/len(l)
    print 'med', sorted(l)[len(l)/2]
    print '1st', sorted(l)[len(l)/4*2]
    print '3rd', sorted(l)[len(l)/4*3]
    print 'min', min(l)
    print 'max', max(l)
Example #39
0
def main():
    member_age = {}
    for rel in Es.query_relations(-1,
                                  Es.by_name('leden'),
                                  None,
                                  None,
                                  deref_who=True):
        if not rel['from']:
            rel['from'] = Es.DT_MIN
        if rel['who'] not in member_age:
            member_age[rel['who']] = Es.DT_MAX
        member_age[rel['who']] = rel['from']
    le = []
    for m, age in six.iteritems(member_age):
        if not m.dateOfBirth:
            continue
        print((age - m.dateOfBirth).days / 365.242, six.text_type(m.name))
        le.append((age - m.dateOfBirth).days / 365.242)
    print('avg', sum(le) / len(le))
    print('med', sorted(le)[len(le) / 2])
    print('1st', sorted(le)[len(le) / 4 * 2])
    print('3rd', sorted(le)[len(le) / 4 * 3])
    print('min', min(le))
    print('max', max(le))
Example #40
0
def update_db(giedo):
    dt_now = now()
    # Load tags
    # TODO cache this
    tags = Es.ids_by_names(
        ('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand'))
    year_overrides = {}
    for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
        year_overrides[t._id] = (t._data['year-override']['type'],
                                 t._data['year-override']['year'])

    # Load _id -> name lut.
    id2name = Es.names_by_ids()
    # Load groups and brands
    groups = Es.of_type_by_name('group')
    groups_set = frozenset(six.itervalues(groups))
    # Find groups that have a virtual group for each year
    year_groups = [g for g in groups_set if tags['!year-group'] in g.tag_ids]

    # Find relations on those groups and add the years for which those
    # relations hold.

    def add_years_to_relations(rels):
        years_of_year_overrides = [
            yo[1] for yo in six.itervalues(year_overrides)
        ]
        until_years = [
            Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX
        ]
        max_until = max(
            max(until_years) if until_years else 0,
            max(years_of_year_overrides) if years_of_year_overrides else 0,
            Es.date_to_year(dt_now))
        from_years = [
            Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN
        ]
        min_from = min(
            min(from_years) if from_years else Es.date_to_year(dt_now),
            Es.date_to_year(dt_now))
        for rel in rels:
            s = (min_from
                 if rel['from'] == DT_MIN else Es.date_to_year(rel['from']))
            t = (max_until
                 if rel['until'] == DT_MAX else Es.date_to_year(rel['until']))
            years = set(range(s, t + 1))
            for tid in rel.get('tags', ()):
                if tid not in year_overrides:
                    continue
                tp, yr = year_overrides[tid]
                if tp:
                    years.add(yr)
                else:
                    if yr not in years:
                        logging.warning('bogus year-override -{} on {}'.format(
                            yr, rel['_id']))
                        continue
                    years.remove(yr)
            rel['years'] = years

    year_group_mrels = tuple(Es.query_relations(_with=year_groups))
    # Check whether all year groups are created
    for g in year_groups:
        mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
        add_years_to_relations(mrels)
        years = set()
        for rel in mrels:
            years.update(rel['years'])
        for year in years:
            n = str(g.name) + str(year)
            if n not in groups:
                logging.info("Creating yeargroup %s" % n)
                _create_yeargroup(g, year, n, tags, groups, id2name)
    # Find all virtual groups
    virtual_groups = [
        g for g in groups_set if tags['!virtual-group'] in g.tag_ids
    ]
    sofa_vgroups = []
    yeargroup_vgroups = []
    for vg in virtual_groups:
        if vg._data['virtual']['type'] == 'sofa':
            sofa_vgroups.append(vg)
        elif vg._data['virtual']['type'] == 'year-group':
            yeargroup_vgroups.append(vg)
        else:
            logging.warn("Unknown vgroup type: %s" %
                         vg._data['virtua']['type'])
    # Find all relations with the sofa virtual groups

    def relkey(rel):
        return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until'])

    vgroup_rlut = dict()
    for rel in Es.query_relations(_with=virtual_groups):
        vgroup_rlut[relkey(rel)] = rel['_id']
    # create look up table of existing sofas
    sofa_queries = []
    sofa_lut = dict()
    for svg in sofa_vgroups:
        w = dict(svg._data['virtual'])
        sofa_queries.append({'how': w['how'], 'with': w['with']})
        k = (w['how'], w['with'])
        if k not in sofa_lut:
            sofa_lut[k] = []
        sofa_lut[k].append(svg)
    # Check whether all year-group relations are in place
    # If there are two relations between an entity and a group in the
    # same year, we do not want this relation to be handled twice.
    # Thus we keep a seperate look-up-table to prevent this.
    year_vgroup_rel_ok = set()
    for mrel in year_group_mrels:
        g = groups[id2name[mrel['with']]]
        for year in mrel['years']:
            yg = groups[str(g.name) + str(year)]
            rrel = {
                'who': mrel['who'],
                'with': yg._id,
                'how': mrel['how'],
                'from': DT_MIN,
                'until': DT_MAX
            }
            if (not relkey(rrel) in vgroup_rlut
                    and relkey(rrel) not in year_vgroup_rel_ok):
                logging.info(
                    "vgroup: adding %s -> %s (%s)" %
                    (id2name[mrel['who']], yg.name, id2name.get(mrel['how'])))
                Es.rcol.insert(rrel)
            elif relkey(rrel) in vgroup_rlut:
                del vgroup_rlut[relkey(rrel)]
            year_vgroup_rel_ok.add(relkey(rrel))
    # Check whether all sofas are created
    sofa_brands = {}
    for b in Es.brands():
        if tags['!sofa-brand'] not in b._data['tags']:
            continue
        sofa_brands[b._id] = b
    for rel in Es.query_relations(how=tuple(sofa_brands.values())):
        if (rel['how'], rel['with']) in sofa_lut:
            continue
        if not id2name[rel['with']]:
            continue
        g = groups[id2name[rel['with']]]
        nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
        logging.info("creating sofa %s" % nm)
        n = {
            'types': ['group', 'tag'],
            'names': [nm],
            'tags': [tags['!virtual-group']],
            'virtual': {
                'type': 'sofa',
                'with': rel['with'],
                'how': rel['how']
            },
            'humanNames': [{
                'name':
                nm,
                'human':
                six.text_type(g.humanName) + ' ' +
                six.text_type(sofa_brands[rel['how']].humanName)
            }]
        }
        n['_id'] = Es.ecol.insert(n)
        groups[nm] = Es.Group(n)
        id2name[n['_id']] = nm
        sofa_vgroups.append(g)
        sofa_lut[rel['how'], rel['with']] = [groups[nm]]
        sofa_queries.append({'how': rel['how'], 'with': rel['with']})
    # Find all relations for the sofa virtual groups and check whether
    # the appropriate relations to the sofas are generated
    for rel in Es.disj_query_relations(sofa_queries):
        for svg in sofa_lut[(rel['how'], rel['with'])]:
            rrel = {
                'how': None,
                'from': rel['from'],
                'until': rel['until'],
                'who': rel['who'],
                'with': svg._id
            }
            if not relkey(rrel) in vgroup_rlut:
                logging.info("sofa: adding %s to %s" %
                             (id2name[rrel['who']], str(svg.name)))
                Es.rcol.insert(rrel)
            else:
                del vgroup_rlut[relkey(rrel)]
    # Check which relations to vgroups are unaccounted for and thus are to
    # be removed.
    for relkey in vgroup_rlut:
        logging.info("removing superfluous %s -> %s (%s)" %
                     (id2name[relkey[0]], id2name.get(
                         relkey[2]), id2name.get(relkey[1])))
        Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
                           relkey[4])
    # Set is_active on Users if and only if they are not in the `leden' group.
    # TODO We might optimize this by including it in a more generic process
    active_users = [
        rel['who'] for rel in Es.by_name('leden').get_rrelated(
            None, dt_now, dt_now, False, False, False)
    ]
    for u in Es.users():
        is_active = u._id in active_users
        if u.is_active == is_active:
            continue
        u._data['is_active'] = is_active
        u.save()
        logging.info("%s user %s",
                     ("activated" if is_active else "deactivated"),
                     str(u.name))
Example #41
0
def generate_wolk_changes(giedo):
    creds = settings.WOLK_MYSQL_SECRET
    if not creds:
        logging.warning('wolk: no credentials available, skipping')
        return None

    todo = {
        'addUser': [],
        'addGroup': [],
        'addUserToGroup': [],
        'removeUserFromGroup': []
    }
    dt_now = now()
    # First, lets see which users and groups to create
    users = dict()  # users to have
    groups = dict()  # groups to have
    missing_groups = set()
    missing_users = set()
    ulut = dict()
    for m in Es.by_name('leden').get_members():
        if not m.got_unix_user:
            continue
        ulut[m._id] = m
        users[str(m.name)] = m
    # Get all groups and create a look-up-table for group membership
    gs = tuple(Es.groups())
    mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
    mlut = dict()
    for g in gs:
        mlut[g._id] = []
    for mrel in mrels:
        mlut[mrel['with']].append(mrel['who'])
    # Flatten out group membership.  For instance: if Giedo is in Kasco
    # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
    # unix group.
    # But first split the mlut graph into a group and a non-group subgraph.
    mlut_g = {}  # { <group> : <members that are groups> }
    mlut_ng = {}  # { <group> : <members that are not groups> }
    for g_id in mlut:
        mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
        mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
    mlut_g_tc = tc(mlut_g)  # transitive closure
    # Generate the { <group> : <indirect non-group members> } graph
    memb_graph = {}
    for g in gs:
        if not g.got_unix_group:
            continue
        memb_graph[g._id] = set(mlut_ng[g._id])
        for h_id in mlut_g_tc[g._id]:
            memb_graph[g._id].update(mlut_ng[h_id])
    # Fill the groups variable
    for g in gs:
        if not g.got_unix_group:
            continue
        groups[str(g.name)] = set(
            [str(ulut[c].name) for c in memb_graph[g._id] if c in ulut])

    # Now, check which users and groups actually exist in owncloud
    missing_users = set(users.keys())
    missing_groups = set(groups.keys())
    dc = pymysql.connect(host=creds[0],
                         user=creds[1],
                         password=creds[2],
                         db=creds[3],
                         charset='utf8')
    try:
        with dc.cursor() as c:
            c.execute("SELECT gid, uid FROM oc_group_user")
            for group, user in c.fetchall():
                if group not in groups:
                    continue
                if user not in users or user not in groups[group]:
                    todo['removeUserFromGroup'].append((user, group))
                    continue
                if user in groups[group]:
                    groups[group].remove(user)
            c.execute("SELECT uid FROM oc_users")
            for user, in c.fetchall():
                if user not in users:
                    logging.info("wolk: stray user %s", user)
                    continue
                missing_users.remove(user)
            c.execute("SELECT gid FROM oc_groups")
            for group, in c.fetchall():
                if group not in groups:
                    logging.info("wolk: stray group %s", user)
                    continue
                missing_groups.remove(group)
            for user in missing_users:
                todo['addUser'].append(
                    (user, six.text_type(users[user].humanName)))
            todo['addGroup'] = list(missing_groups)
            for group, missing_members in six.iteritems(groups):
                for user in missing_members:
                    todo['addUserToGroup'].append((user, group))
    finally:
        dc.close()
    return todo
Example #42
0
def update_db(giedo):
        dt_now = now()
        # Load tags
        # TODO cache this
        tags = Es.ids_by_names(('!year-group', '!year-overrides',
                        '!virtual-group', '!sofa-brand'))
        year_overrides = {}
        for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
                year_overrides[t._id] = (t._data['year-override']['type'],
                                          t._data['year-override']['year'])

        # Load _id -> name lut.
        id2name = Es.names_by_ids()
        # Load groups and brands
        groups = Es.of_type_by_name('group')
        groups_set = frozenset(groups.values())
        # Find groups that have a virtual group for each year
        year_groups = [g for g in groups.itervalues()
                        if tags['!year-group'] in g.tag_ids]
        # Find relations on those groups and add the years for which those
        # relations hold.
        def add_years_to_relations(rels):
                until_years = [Es.date_to_year(r['until']) for r in rels
                                        if r['until'] != DT_MAX]
                max_until = max(max(until_years) if until_years else 0,
                                Es.date_to_year(dt_now))
                from_years = [Es.date_to_year(r['from']) for r in rels
                                        if r['from'] != DT_MIN]
                min_from = min(min(from_years) if from_years else
                                Es.date_to_year(dt_now),
                                Es.date_to_year(dt_now))
                for rel in rels:
                        s = min_from if rel['from'] == DT_MIN \
                                        else Es.date_to_year(rel['from'])
                        t = max_until if rel['until'] == DT_MAX \
                                        else Es.date_to_year(rel['until'])
                        years = set(range(s, t+1))
                        for tid in rel.get('tags', ()):
                                if tid not in year_overrides:
                                        continue
                                yr, tp = year_overrides[tid]
                                if tp:
                                        years.add(yr)
                                else:
                                        years.remove(yr)
                        rel['years'] = years
        year_group_mrels = tuple(Es.query_relations(
                _with=year_groups))
        # Check whether all year groups are created
        for g in year_groups:
                mrels = filter(lambda x: x['with']==g._id, year_group_mrels)
                add_years_to_relations(mrels)
                years = set()
                for rel in mrels:
                        years.update(rel['years'])
                for year in years:
                        n = str(g.name) + str(year)
                        if n not in groups:
                                logging.info("Creating yeargroup %s" % n)
                                _create_yeargroup(g, year, n, tags, groups,
                                               id2name)
        # Find all virtual groups
        virtual_groups = [g for g in groups_set
                        if tags['!virtual-group'] in g.tag_ids]
        sofa_vgroups = []
        yeargroup_vgroups = []
        for vg in virtual_groups:
                if vg._data['virtual']['type'] == 'sofa':
                        sofa_vgroups.append(vg)
                elif vg._data['virtual']['type'] == 'year-group':
                        yeargroup_vgroups.append(vg)
                else:
                        logging.warn("Unknown vgroup type: %s" \
                                        % vg._data['virtua']['type'])
        # Find all relations with the sofa virtual groups
        def relkey(rel):
                return (rel['who'], rel['how'], rel['with'],
                                rel['from'], rel['until'])
        vgroup_rlut = dict()
        for rel in Es.query_relations(_with=virtual_groups):
                vgroup_rlut[relkey(rel)] = rel['_id']
        # create look up table of existing sofas
        sofa_queries = []
        sofa_lut = dict()
        for svg in sofa_vgroups:
                w = dict(svg._data['virtual'])
                sofa_queries.append({'how': w['how'],
                                     'with': w['with']})
                k = (w['how'], w['with'])
                if k not in sofa_lut:
                        sofa_lut[k] = []
                sofa_lut[k].append(svg)
        # Check whether all year-group relations are in place
        for mrel in year_group_mrels:
                g = groups[id2name[mrel['with']]]
                for year in mrel['years']:
                        yg = groups[str(g.name) + str(year)]
                        rrel = {'who': mrel['who'],
                                'with': yg._id,
                                'how': mrel['how'],
                                'from': DT_MIN,
                                'until': DT_MAX}
                        if not relkey(rrel) in vgroup_rlut:
                                logging.info("vgroup: adding %s -> %s (%s)" % (
                                                id2name[mrel['who']], yg.name,
                                                id2name.get(mrel['how'])))
                                Es.rcol.insert(rrel)
                        else:
                                del vgroup_rlut[relkey(rrel)]
        # Check whether all sofas are created
        sofa_brands = {}
        for b in Es.brands():
                if tags['!sofa-brand'] not in b._data['tags']:
                        continue
                sofa_brands[b._id] = b
        for rel in Es.query_relations(how=sofa_brands.values()):
                if (rel['how'], rel['with']) in sofa_lut:
                        continue
                g = groups[id2name[rel['with']]]
                nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
                logging.info("creating sofa %s" % nm)
                n = {'types': ['group','tag'],
                     'names': [nm],
                     'tags': [tags['!virtual-group']],
                     'virtual': {
                             'type': 'sofa',
                             'with': rel['with'],
                             'how': rel['how']},
                     'humanNames': [{
                             'name': nm,
                             'human': unicode(g.humanName) + ' ' +
                                unicode(sofa_brands[rel['how']].humanName)}]}
                n['_id'] = Es.ecol.insert(n)
                groups[nm] = Es.Group(n)
                id2name[n['_id']] = nm
                sofa_vgroups.append(g)
                sofa_lut[rel['how'], rel['with']] = [groups[nm]]
                sofa_queries.append({'how': rel['how'],
                                     'with': rel['with']})
        # Find all relations for the sofa virtual groups and check whether
        # the appropriate relations to the sofas are generated
        for rel in Es.disj_query_relations(sofa_queries):
                for svg in sofa_lut[(rel['how'], rel['with'])]:
                        rrel = {'how': None,
                                'from': rel['from'],
                                'until': rel['until'],
                                'who': rel['who'],
                                'with': svg._id}
                        if not relkey(rrel) in vgroup_rlut:
                                logging.info("sofa: adding %s to %s" % (
                                        id2name[rrel['who']], str(svg.name)))
                                Es.rcol.insert(rrel)
                        else:
                                del vgroup_rlut[relkey(rrel)]
        # Check which relations to vgroups are unaccounted for and thus are to
        # be removed.
        for relkey in vgroup_rlut:
                logging.info("removing superfluous %s -> %s (%s)" % (
                        id2name[relkey[0]], id2name.get(relkey[2]),
                        id2name.get(relkey[1])))
Example #43
0
File: db.py Project: aykevl/kninfra
def update_db(giedo):
    dt_now = now()
    # Load tags
    # TODO cache this
    tags = Es.ids_by_names(('!year-group', '!year-overrides',
                            '!virtual-group', '!sofa-brand'))
    year_overrides = {}
    for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
        year_overrides[t._id] = (t._data['year-override']['type'],
                                 t._data['year-override']['year'])

    # Load _id -> name lut.
    id2name = Es.names_by_ids()
    # Load groups and brands
    groups = Es.of_type_by_name('group')
    groups_set = frozenset(six.itervalues(groups))
    # Find groups that have a virtual group for each year
    year_groups = [g for g in groups_set
                   if tags['!year-group'] in g.tag_ids]
    # Find relations on those groups and add the years for which those
    # relations hold.

    def add_years_to_relations(rels):
        years_of_year_overrides = [yo[1] for yo
                                   in six.itervalues(year_overrides)]
        until_years = [Es.date_to_year(r['until']) for r in rels
                       if r['until'] != DT_MAX]
        max_until = max(
            max(until_years) if until_years else 0,
            max(years_of_year_overrides) if years_of_year_overrides else 0,
            Es.date_to_year(dt_now)
        )
        from_years = [Es.date_to_year(r['from']) for r in rels
                      if r['from'] != DT_MIN]
        min_from = min(min(from_years) if from_years else
                       Es.date_to_year(dt_now),
                       Es.date_to_year(dt_now))
        for rel in rels:
            s = (min_from if rel['from'] == DT_MIN
                 else Es.date_to_year(rel['from']))
            t = (max_until if rel['until'] == DT_MAX
                 else Es.date_to_year(rel['until']))
            years = set(range(s, t + 1))
            for tid in rel.get('tags', ()):
                if tid not in year_overrides:
                    continue
                tp, yr = year_overrides[tid]
                if tp:
                    years.add(yr)
                else:
                    if yr not in years:
                        logging.warning('bogus year-override -{} on {}'.format(
                            yr, rel['_id']))
                        continue
                    years.remove(yr)
            rel['years'] = years
    year_group_mrels = tuple(Es.query_relations(_with=year_groups))
    # Check whether all year groups are created
    for g in year_groups:
        mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
        add_years_to_relations(mrels)
        years = set()
        for rel in mrels:
            years.update(rel['years'])
        for year in years:
            n = str(g.name) + str(year)
            if n not in groups:
                logging.info("Creating yeargroup %s" % n)
                _create_yeargroup(g, year, n, tags, groups, id2name)
    # Find all virtual groups
    virtual_groups = [g for g in groups_set
                      if tags['!virtual-group'] in g.tag_ids]
    sofa_vgroups = []
    yeargroup_vgroups = []
    for vg in virtual_groups:
        if vg._data['virtual']['type'] == 'sofa':
            sofa_vgroups.append(vg)
        elif vg._data['virtual']['type'] == 'year-group':
            yeargroup_vgroups.append(vg)
        else:
            logging.warn("Unknown vgroup type: %s"
                         % vg._data['virtua']['type'])
    # Find all relations with the sofa virtual groups

    def relkey(rel):
        return (rel['who'], rel['how'], rel['with'],
                rel['from'], rel['until'])
    vgroup_rlut = dict()
    for rel in Es.query_relations(_with=virtual_groups):
        vgroup_rlut[relkey(rel)] = rel['_id']
    # create look up table of existing sofas
    sofa_queries = []
    sofa_lut = dict()
    for svg in sofa_vgroups:
        w = dict(svg._data['virtual'])
        sofa_queries.append({'how': w['how'],
                             'with': w['with']})
        k = (w['how'], w['with'])
        if k not in sofa_lut:
            sofa_lut[k] = []
        sofa_lut[k].append(svg)
    # Check whether all year-group relations are in place
    # If there are two relations between an entity and a group in the
    # same year, we do not want this relation to be handled twice.
    # Thus we keep a seperate look-up-table to prevent this.
    year_vgroup_rel_ok = set()
    for mrel in year_group_mrels:
        g = groups[id2name[mrel['with']]]
        for year in mrel['years']:
            yg = groups[str(g.name) + str(year)]
            rrel = {'who': mrel['who'],
                    'with': yg._id,
                    'how': mrel['how'],
                    'from': DT_MIN,
                    'until': DT_MAX}
            if (not relkey(rrel) in vgroup_rlut and relkey(rrel)
                    not in year_vgroup_rel_ok):
                logging.info("vgroup: adding %s -> %s (%s)" % (
                    id2name[mrel['who']], yg.name,
                    id2name.get(mrel['how'])))
                Es.rcol.insert(rrel)
            elif relkey(rrel) in vgroup_rlut:
                del vgroup_rlut[relkey(rrel)]
            year_vgroup_rel_ok.add(relkey(rrel))
    # Check whether all sofas are created
    sofa_brands = {}
    for b in Es.brands():
        if tags['!sofa-brand'] not in b._data['tags']:
            continue
        sofa_brands[b._id] = b
    for rel in Es.query_relations(how=tuple(sofa_brands.values())):
        if (rel['how'], rel['with']) in sofa_lut:
            continue
        if not id2name[rel['with']]:
            continue
        g = groups[id2name[rel['with']]]
        nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
        logging.info("creating sofa %s" % nm)
        n = {'types': ['group', 'tag'],
             'names': [nm],
             'tags': [tags['!virtual-group']],
             'virtual': {
                 'type': 'sofa',
                 'with': rel['with'],
                 'how': rel['how']},
             'humanNames': [{
                 'name': nm,
                 'human': six.text_type(g.humanName) + ' ' +
                 six.text_type(sofa_brands[rel['how']].humanName)}]}
        n['_id'] = Es.ecol.insert(n)
        groups[nm] = Es.Group(n)
        id2name[n['_id']] = nm
        sofa_vgroups.append(g)
        sofa_lut[rel['how'], rel['with']] = [groups[nm]]
        sofa_queries.append({'how': rel['how'],
                             'with': rel['with']})
    # Find all relations for the sofa virtual groups and check whether
    # the appropriate relations to the sofas are generated
    for rel in Es.disj_query_relations(sofa_queries):
        for svg in sofa_lut[(rel['how'], rel['with'])]:
            rrel = {'how': None,
                    'from': rel['from'],
                    'until': rel['until'],
                    'who': rel['who'],
                    'with': svg._id}
            if not relkey(rrel) in vgroup_rlut:
                logging.info("sofa: adding %s to %s" % (
                    id2name[rrel['who']], str(svg.name)))
                Es.rcol.insert(rrel)
            else:
                del vgroup_rlut[relkey(rrel)]
    # Check which relations to vgroups are unaccounted for and thus are to
    # be removed.
    for relkey in vgroup_rlut:
        logging.info("removing superfluous %s -> %s (%s)" % (
            id2name[relkey[0]], id2name.get(relkey[2]),
            id2name.get(relkey[1])))
        Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
                           relkey[4])
    # Set is_active on Users if and only if they are not in the `leden' group.
    # TODO We might optimize this by including it in a more generic process
    active_users = [rel['who'] for rel in Es.by_name('leden').get_rrelated(
        None, dt_now, dt_now, False, False, False)]
    for u in Es.users():
        is_active = u._id in active_users
        if u.is_active == is_active:
            continue
        u._data['is_active'] = is_active
        u.save()
        logging.info(
            "%s user %s",
            ("activated" if is_active else "deactivated"),
            str(u.name)
        )
Example #44
0
llut = dict()
clut = dict()
comms = [g for g in Es.by_name('comms').get_bearers() if g.is_group]
comms.append(Es.by_name('bestuur'))
leden = Es.by_name('leden').get_members()
leden_lut = {}
comms_lut = {}
for c in comms:
    comms_lut[_id(c)] = c
for l in leden:
    clut[l] = (DT_MAX, None)
    llut[l] = DT_MAX
    leden_lut[_id(l)] = l

for rel in Es.query_relations(who=leden, _with=Es.by_name('leden'), how=None):
    w = leden_lut[rel['who']]
    llut[w] = min(llut[w], rel['from'])

for rel in Es.query_relations(who=leden, _with=comms, how=None):
    w = leden_lut[rel['who']]
    if rel['from'] < clut[w][0]:
        clut[w] = (rel['from'], comms_lut[rel['with']])

had = set()
for y in range(3, 10):
    print()
    print('Jaar', y)
    nnever = 0
    nalways = 0
    navg = 0
Example #45
0
def generate_wolk_changes(giedo):
    creds = settings.WOLK_MYSQL_SECRET
    if not creds:
        logging.warning('wolk: no credentials available, skipping')
        return None

    todo = {'addUser': [], 'addGroup': [],
            'addUserToGroup': [], 'removeUserFromGroup': []}
    dt_now = now()
    # First, lets see which users and groups to create
    users = dict()      # users to have
    groups = dict()     # groups to have
    missing_groups = set()
    missing_users = set()
    ulut = dict()
    for m in Es.by_name('leden').get_members():
        if not m.got_unix_user:
            continue
        ulut[m._id] = m
        users[str(m.name)] = m
    # Get all groups and create a look-up-table for group membership
    gs = tuple(Es.groups())
    mrels = Es.query_relations(how=None, _with=gs, _from=dt_now, until=dt_now)
    mlut = dict()
    for g in gs:
        mlut[g._id] = []
    for mrel in mrels:
        mlut[mrel['with']].append(mrel['who'])
    # Flatten out group membership.  For instance: if Giedo is in Kasco
    # and Kasco is in Boekenlezers, then Giedo is also in the Boekenlezers
    # unix group.
    # But first split the mlut graph into a group and a non-group subgraph.
    mlut_g = {}     # { <group> : <members that are groups> }
    mlut_ng = {}    # { <group> : <members that are not groups> }
    for g_id in mlut:
        mlut_g[g_id] = [c for c in mlut[g_id] if c in mlut]
        mlut_ng[g_id] = [c for c in mlut[g_id] if c not in mlut]
    mlut_g_tc = tc(mlut_g)  # transitive closure
    # Generate the { <group> : <indirect non-group members> } graph
    memb_graph = {}
    for g in gs:
        if not g.got_unix_group:
            continue
        memb_graph[g._id] = set(mlut_ng[g._id])
        for h_id in mlut_g_tc[g._id]:
            memb_graph[g._id].update(mlut_ng[h_id])
    # Fill the groups variable
    for g in gs:
        if not g.got_unix_group:
            continue
        groups[str(g.name)] = set([str(ulut[c].name)
                                   for c in memb_graph[g._id] if c in ulut])

    # Now, check which users and groups actually exist in owncloud
    missing_users = set(users.keys())
    missing_groups = set(groups.keys())
    dc = pymysql.connect(
        host=creds[0],
        user=creds[1],
        password=creds[2],
        db=creds[3],
        charset='utf8'
    )
    try:
        with dc.cursor() as c:
            c.execute("SELECT gid, uid FROM oc_group_user")
            for group, user in c.fetchall():
                if group not in groups:
                    continue
                if user not in users or user not in groups[group]:
                    todo['removeUserFromGroup'].append((user, group))
                    continue
                if user in groups[group]:
                    groups[group].remove(user)
            c.execute("SELECT uid FROM oc_users")
            for user, in c.fetchall():
                if user not in users:
                    logging.info("wolk: stray user %s", user)
                    continue
                missing_users.remove(user)
            c.execute("SELECT gid FROM oc_groups")
            for group, in c.fetchall():
                if group not in groups:
                    logging.info("wolk: stray group %s", user)
                    continue
                missing_groups.remove(group)
            for user in missing_users:
                todo['addUser'].append((
                    user,
                    six.text_type(users[user].humanName)
                ))
            todo['addGroup'] = list(missing_groups)
            for group, missing_members in six.iteritems(groups):
                for user in missing_members:
                    todo['addUserToGroup'].append((user, group))
    finally:
        dc.close()
    return todo