def set_tags(self, tags, save=True): ''' Set tags by their usernames. ''' name2id = Es.ids_by_names(tags) self._data['tags'] = [] for name in tags: self._data['tags'].append(name2id[name]) if save: self.save()
def set_tags(self, tags, save=True): """ Set tags by their usernames. """ name2id = Es.ids_by_names(tags) self._data["tags"] = [] for name in tags: self._data["tags"].append(name2id[name]) self.update_search_text(save=False) if save: self.save()
def set_tags(self, tags, save=True): ''' Set tags by their usernames. ''' name2id = Es.ids_by_names(tags) self._data['tags'] = [] for name in tags: self._data['tags'].append(name2id[name]) self.update_search_text(save=False) if save: self.save()
def main(data): def str_to_date(s): if s is None: return None return datetime(*strptime(s,'%Y-%m-%d')[:3]) def year_to_dates(year): return (datetime(2003+year,9,1), datetime(2004+year,8,31)) def create_tag(name, humanName, tags=[]): return Es.ecol.insert({'types': ['tag'], 'names': [name], 'humanNames': [{ 'name': name, 'human': humanName}], 'tags': tags}) print 'dropping' Es.ecol.drop() Es.rcol.drop() Es.mcol.drop() subscr_Es.ecol.drop() subscr_Es.scol.drop() print 'creating indices' Es.ensure_indices() subscr_Es.ensure_indices() mod_Es.ensure_indices() conv_inst = dict() conv_study = dict() conv_group = dict() conv_group_byname = dict() conv_event = dict() conv_seat = dict() conv_user = dict() ignore_groups = frozenset(['leden-oud']) ignore_groups_members = frozenset(['leden']) ignore_groups_ids = set() ignore_groups_members_ids = set() year_groups = frozenset( ['leden'+str(x) for x in range(1,9)]+ ['kasco'+str(x) for x in range(1,9)]+ ['bestuur'+str(x) for x in range(1,9)]) year_groups_ids = dict() year_groups_lut = {} print 'initial tags' system_tag = create_tag('!system', 'Systeemstempels') year_overrides_tag = create_tag('!year-overrides', 'Jaarlidmaatschapstempels', [system_tag]) virtual_group_tag = create_tag("!virtual-group", 'Virtuele groep', [system_tag]) sofa_brand_tag = create_tag("!sofa-brand", 'Sofa merk', [system_tag]) year_group_tag = create_tag("!year-group", 'Jaargroep', [system_tag]) for i in xrange(1,9): Es.ecol.insert({'types': ['tag'], 'humanNames': [{'human': 'Wel jaar %s' % i}], 'year-override': {'year': i, 'type': True}, 'tags': [year_overrides_tag]}) Es.ecol.insert({'types': ['tag'], 'humanNames': [{'human': 'Niet jaar %s' % i}], 'year-override': {'year': i, 'type': False}, 'tags': [year_overrides_tag]}) print 'institutes' for m in data['EduInstitute']: n = { 'types': ['institute'], 'humanNames': [{'human': m['name']}]} conv_inst[m['id']] = Es.ecol.insert(n) print 'studies' for m in data['Study']: n = { 'types': ['study'], 'humanNames': [{'human': m['name']}]} conv_study[m['id']] = Es.ecol.insert(n) print 'initial groups' conv_group_byname['bestuur'] = { 'id': Es.ecol.insert({ 'types': ['group'], 'names': ['bestuur'], 'tags': [year_group_tag], 'humanNames': [{ 'name': 'bestuur', 'human': 'Bestuur', 'genitive_prefix': 'van het'}], 'description': "Het bestuur"}), 'name': 'bestuur', 'humanName': 'Bestuur'} conv_group_byname['kasco'] = { 'id': Es.ecol.insert({ 'types': ['group'], 'names': ['kasco'], 'tags': [year_group_tag], 'humanNames': [{ 'name': 'kasco', 'human': 'Kascontrolecommissie', 'genitive_prefix': 'van de'}], 'description': "De kascontrolecommissie"}), 'name': 'kasco', 'humanName': 'Bestuur'} print 'groups' for m in data['OldKnGroup']: if m['name'] in ignore_groups: ignore_groups_ids.add(m['id']) continue if m['name'] in ignore_groups_members: ignore_groups_members_ids.add(m['id']) if m['name'] in year_groups: year_groups_ids[m['id']] = m['name'] group = m['name'][:-1] year = int(m['name'][-1:]) year_groups_lut[m['id']] = (group, year) continue if m['name'] == 'leden': m['isVirtual'] = False # fix for leden n = { 'types': ['tag' if m['isVirtual'] else 'group'], 'names': [m['name']], 'humanNames': [{ 'name': m['name'], 'human': m['humanName'], 'genitive_prefix': m['genitive_prefix'] }], 'description': m['description'], 'temp':{ 'is_virtual': m['isVirtual'] } } conv_group[m['id']] = {'id': Es.ecol.insert(n), 'name': m['name'], 'humanName': m['humanName']} conv_group_byname[m['name']] = conv_group[m['id']] print 'group hierarchy' for m in data['OldKnGroup']: if m['name'] in year_groups or m['name'] in ignore_groups: continue if m['parent'] is not None: if not m['parent'] in conv_group: print " %s was orphaned" % m['name'] continue Es.ecol.update({'_id': conv_group[m['id']]['id']}, {'$push': {'tags': conv_group[ m['parent']]['id']}}) print 'users' for m in data['OldKnUser']: bits = m['password'].split('$') if len(bits) == 3: pwd = {'algorithm': bits[0], 'salt': bits[1], 'hash': bits[2]} else: pwd = None n = { 'types': ['user'], 'names': [m['username']], 'humanNames': [{'human': m['first_name'] + ' ' + m['last_name']}], 'person': { 'titles': [], 'nick': m['first_name'], 'given': None, 'family': m['last_name'], 'gender': m['gender'], 'dateOfBirth': str_to_date(m['dateOfBirth']) }, 'emailAddresses': [ {'email': m['email'], 'from': DT_MIN, 'until': DT_MAX }], 'addresses': [ {'street': m['addr_street'], 'number': m['addr_number'], 'zip': m['addr_zipCode'], 'city': m['addr_city'], 'from': DT_MIN, 'until': DT_MAX }], 'telephones': [ {'number': m['telephone'], 'from': DT_MIN, 'until': DT_MAX}], 'studies': [ {'institute': conv_inst.get(m['institute']), 'study': conv_study.get(m['study']), 'from': DT_MIN, 'until': DT_MAX, 'number': m['studentNumber']} ], 'temp': { 'oud': m['in_oud'], 'aan': m['in_aan'], 'incasso': m['got_incasso'], 'joined': m['dateJoined'], 'remarks': m['remarks'] }, 'is_active': m['is_active'], 'password': pwd } conv_user[m['id']] = Es.ecol.insert(n) for g in m['groups']: if g in ignore_groups_ids or \ g in ignore_groups_members_ids: continue if g in year_groups_ids: gname, year = year_groups_lut[g] f, u = year_to_dates(year) Es.rcol.insert({ 'with': conv_group_byname[gname]['id'], 'who': conv_user[m['id']], 'from': f, 'until': u, 'how': None}) continue Es.rcol.insert({ 'with': conv_group[g]['id'], 'who': conv_user[m['id']], 'from': DT_MIN, 'until': DT_MAX, 'how': None}) print 'brands' for m in data['OldSeat']: if m['name'] == 'deelhoofd': m['name'] = 'graficideelhoofd' if m['name'] in conv_seat: continue n = {'types': ['brand'], 'names': [m['name']], 'tags': [sofa_brand_tag], 'humanNames': [{'name': m['name'], 'human': m['humanName']}]} conv_seat[m['name']] = {'id': Es.ecol.insert(n)} print 'seats' for m in data['OldSeat']: if m['group'] in year_groups_ids: gname = year_groups_ids[m['group']] gdat = conv_group_byname[gname[:-1]] _from, until = year_to_dates(int(gname[-1:])) else: gdat = conv_group[m['group']] _from, until = DT_MIN, DT_MAX n = {'types': ['group'], 'names': [gdat['name'] + '-' + m['name']], 'description': [m['description']], 'tags': [virtual_group_tag], 'virtual': { 'type': 'sofa', 'with': gdat['id'], 'how': conv_seat[m['name']]['id']}, 'humanNames': [{ 'name': gdat['name'] + '-' + m['name'], 'human': m['humanName'] + ' ' + gdat['humanName']}]} i = Es.ecol.insert(n) Es.rcol.insert({'who': conv_user[m['user']], 'from': _from, 'until': until, 'how': conv_seat[m['name']]['id'], 'with': gdat['id']}) print 'merging relations' print ' list until' lut = dict() plan_changes = dict() plan_remove = set() for r in Es.rcol.find({'until': {'$lt': DT_MAX}}): lut[r['until'] + timedelta(1,0), r['with'], r['how'], r['who']] = r['_id'] print ' crossreference from' for r in Es.rcol.find({'from': {'$gt': DT_MIN}}): n = (r['from'], r['with'], r['how'], r['who']) if n not in lut: continue plan_changes[lut[n]] = (r['until'], r['_id']) plan_remove.add(r['_id']) print ' transitive closure of plan' print 'small final tweaks to groups' Es.ecol.update({'names': 'leden'}, {'$push': {'tags': year_group_tag}}) done = False while not done: done = True for k,v in plan_changes.iteritems(): if v[1] in plan_changes: plan_changes[k] = plan_changes[v[1]] del plan_changes[v[1]] done = False break print ' execute' for r in plan_remove: Es.rcol.remove({'_id': r}) for k,v in plan_changes.iteritems(): Es.rcol.update({'_id': k}, {'$set': {'until': v[0]}}) print 'event' for m in data['Event']: if m['owner'] not in conv_group: gname = year_groups_lut[m['owner']][0] gid = conv_group_byname[gname]['id'] else: gid = conv_group[m['owner']]['id'] conv_event[m['id']] = subscr_Es.ecol.insert({ 'mailBody': m['mailBody'], 'humanName': m['humanName'], 'description': m['description'], 'cost': m['cost'], 'is_open': m['is_open'], 'owner': gid, 'name': m['name']}) print 'event subscriptions' for m in data['EventSubscription']: subscr_Es.scol.insert({ 'event': conv_event[m['event']], 'userNotes': m['userNotes'], 'debit': m['debit'], 'user': conv_user[m['user']]}) print 'giedo updatedb' update_db(None) print 'alias' print ' ids_by_names' name2id = Es.ids_by_names() print ' to graph' alias_graph = {} for m in data['Alias']: alias_graph[m['source']] = m['target'] print ' tarjan' for scc in tarjan(alias_graph): assert len(scc) == 1 src = scc[0] if src in name2id: continue if not src in alias_graph: continue if not alias_graph[src] in name2id: print ' ? %s -> %s' % (src, alias_graph[src]) continue name2id[src] = name2id[alias_graph[src]] Es.ecol.update({'names': alias_graph[src]}, {'$push': {'names': src}})
def update_db(giedo): dt_now = now() # Load tags # TODO cache this tags = Es.ids_by_names( ('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand')) year_overrides = {} for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag): year_overrides[t._id] = (t._data['year-override']['type'], t._data['year-override']['year']) # Load _id -> name lut. id2name = Es.names_by_ids() # Load groups and brands groups = Es.of_type_by_name('group') groups_set = frozenset(six.itervalues(groups)) # Find groups that have a virtual group for each year year_groups = [g for g in groups_set if tags['!year-group'] in g.tag_ids] # Find relations on those groups and add the years for which those # relations hold. def add_years_to_relations(rels): years_of_year_overrides = [ yo[1] for yo in six.itervalues(year_overrides) ] until_years = [ Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX ] max_until = max( max(until_years) if until_years else 0, max(years_of_year_overrides) if years_of_year_overrides else 0, Es.date_to_year(dt_now)) from_years = [ Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN ] min_from = min( min(from_years) if from_years else Es.date_to_year(dt_now), Es.date_to_year(dt_now)) for rel in rels: s = (min_from if rel['from'] == DT_MIN else Es.date_to_year(rel['from'])) t = (max_until if rel['until'] == DT_MAX else Es.date_to_year(rel['until'])) years = set(range(s, t + 1)) for tid in rel.get('tags', ()): if tid not in year_overrides: continue tp, yr = year_overrides[tid] if tp: years.add(yr) else: if yr not in years: logging.warning('bogus year-override -{} on {}'.format( yr, rel['_id'])) continue years.remove(yr) rel['years'] = years year_group_mrels = tuple(Es.query_relations(_with=year_groups)) # Check whether all year groups are created for g in year_groups: mrels = [rel for rel in year_group_mrels if rel['with'] == g._id] add_years_to_relations(mrels) years = set() for rel in mrels: years.update(rel['years']) for year in years: n = str(g.name) + str(year) if n not in groups: logging.info("Creating yeargroup %s" % n) _create_yeargroup(g, year, n, tags, groups, id2name) # Find all virtual groups virtual_groups = [ g for g in groups_set if tags['!virtual-group'] in g.tag_ids ] sofa_vgroups = [] yeargroup_vgroups = [] for vg in virtual_groups: if vg._data['virtual']['type'] == 'sofa': sofa_vgroups.append(vg) elif vg._data['virtual']['type'] == 'year-group': yeargroup_vgroups.append(vg) else: logging.warn("Unknown vgroup type: %s" % vg._data['virtua']['type']) # Find all relations with the sofa virtual groups def relkey(rel): return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until']) vgroup_rlut = dict() for rel in Es.query_relations(_with=virtual_groups): vgroup_rlut[relkey(rel)] = rel['_id'] # create look up table of existing sofas sofa_queries = [] sofa_lut = dict() for svg in sofa_vgroups: w = dict(svg._data['virtual']) sofa_queries.append({'how': w['how'], 'with': w['with']}) k = (w['how'], w['with']) if k not in sofa_lut: sofa_lut[k] = [] sofa_lut[k].append(svg) # Check whether all year-group relations are in place # If there are two relations between an entity and a group in the # same year, we do not want this relation to be handled twice. # Thus we keep a seperate look-up-table to prevent this. year_vgroup_rel_ok = set() for mrel in year_group_mrels: g = groups[id2name[mrel['with']]] for year in mrel['years']: yg = groups[str(g.name) + str(year)] rrel = { 'who': mrel['who'], 'with': yg._id, 'how': mrel['how'], 'from': DT_MIN, 'until': DT_MAX } if (not relkey(rrel) in vgroup_rlut and relkey(rrel) not in year_vgroup_rel_ok): logging.info( "vgroup: adding %s -> %s (%s)" % (id2name[mrel['who']], yg.name, id2name.get(mrel['how']))) Es.rcol.insert(rrel) elif relkey(rrel) in vgroup_rlut: del vgroup_rlut[relkey(rrel)] year_vgroup_rel_ok.add(relkey(rrel)) # Check whether all sofas are created sofa_brands = {} for b in Es.brands(): if tags['!sofa-brand'] not in b._data['tags']: continue sofa_brands[b._id] = b for rel in Es.query_relations(how=tuple(sofa_brands.values())): if (rel['how'], rel['with']) in sofa_lut: continue if not id2name[rel['with']]: continue g = groups[id2name[rel['with']]] nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix logging.info("creating sofa %s" % nm) n = { 'types': ['group', 'tag'], 'names': [nm], 'tags': [tags['!virtual-group']], 'virtual': { 'type': 'sofa', 'with': rel['with'], 'how': rel['how'] }, 'humanNames': [{ 'name': nm, 'human': six.text_type(g.humanName) + ' ' + six.text_type(sofa_brands[rel['how']].humanName) }] } n['_id'] = Es.ecol.insert(n) groups[nm] = Es.Group(n) id2name[n['_id']] = nm sofa_vgroups.append(g) sofa_lut[rel['how'], rel['with']] = [groups[nm]] sofa_queries.append({'how': rel['how'], 'with': rel['with']}) # Find all relations for the sofa virtual groups and check whether # the appropriate relations to the sofas are generated for rel in Es.disj_query_relations(sofa_queries): for svg in sofa_lut[(rel['how'], rel['with'])]: rrel = { 'how': None, 'from': rel['from'], 'until': rel['until'], 'who': rel['who'], 'with': svg._id } if not relkey(rrel) in vgroup_rlut: logging.info("sofa: adding %s to %s" % (id2name[rrel['who']], str(svg.name))) Es.rcol.insert(rrel) else: del vgroup_rlut[relkey(rrel)] # Check which relations to vgroups are unaccounted for and thus are to # be removed. for relkey in vgroup_rlut: logging.info("removing superfluous %s -> %s (%s)" % (id2name[relkey[0]], id2name.get( relkey[2]), id2name.get(relkey[1]))) Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3], relkey[4]) # Set is_active on Users if and only if they are not in the `leden' group. # TODO We might optimize this by including it in a more generic process active_users = [ rel['who'] for rel in Es.by_name('leden').get_rrelated( None, dt_now, dt_now, False, False, False) ] for u in Es.users(): is_active = u._id in active_users if u.is_active == is_active: continue u._data['is_active'] = is_active u.save() logging.info("%s user %s", ("activated" if is_active else "deactivated"), str(u.name))
def update_db(giedo): dt_now = now() # Load tags # TODO cache this tags = Es.ids_by_names(('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand')) year_overrides = {} for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag): year_overrides[t._id] = (t._data['year-override']['type'], t._data['year-override']['year']) # Load _id -> name lut. id2name = Es.names_by_ids() # Load groups and brands groups = Es.of_type_by_name('group') groups_set = frozenset(six.itervalues(groups)) # Find groups that have a virtual group for each year year_groups = [g for g in groups_set if tags['!year-group'] in g.tag_ids] # Find relations on those groups and add the years for which those # relations hold. def add_years_to_relations(rels): years_of_year_overrides = [yo[1] for yo in six.itervalues(year_overrides)] until_years = [Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX] max_until = max( max(until_years) if until_years else 0, max(years_of_year_overrides) if years_of_year_overrides else 0, Es.date_to_year(dt_now) ) from_years = [Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN] min_from = min(min(from_years) if from_years else Es.date_to_year(dt_now), Es.date_to_year(dt_now)) for rel in rels: s = (min_from if rel['from'] == DT_MIN else Es.date_to_year(rel['from'])) t = (max_until if rel['until'] == DT_MAX else Es.date_to_year(rel['until'])) years = set(range(s, t + 1)) for tid in rel.get('tags', ()): if tid not in year_overrides: continue tp, yr = year_overrides[tid] if tp: years.add(yr) else: if yr not in years: logging.warning('bogus year-override -{} on {}'.format( yr, rel['_id'])) continue years.remove(yr) rel['years'] = years year_group_mrels = tuple(Es.query_relations(_with=year_groups)) # Check whether all year groups are created for g in year_groups: mrels = [rel for rel in year_group_mrels if rel['with'] == g._id] add_years_to_relations(mrels) years = set() for rel in mrels: years.update(rel['years']) for year in years: n = str(g.name) + str(year) if n not in groups: logging.info("Creating yeargroup %s" % n) _create_yeargroup(g, year, n, tags, groups, id2name) # Find all virtual groups virtual_groups = [g for g in groups_set if tags['!virtual-group'] in g.tag_ids] sofa_vgroups = [] yeargroup_vgroups = [] for vg in virtual_groups: if vg._data['virtual']['type'] == 'sofa': sofa_vgroups.append(vg) elif vg._data['virtual']['type'] == 'year-group': yeargroup_vgroups.append(vg) else: logging.warn("Unknown vgroup type: %s" % vg._data['virtua']['type']) # Find all relations with the sofa virtual groups def relkey(rel): return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until']) vgroup_rlut = dict() for rel in Es.query_relations(_with=virtual_groups): vgroup_rlut[relkey(rel)] = rel['_id'] # create look up table of existing sofas sofa_queries = [] sofa_lut = dict() for svg in sofa_vgroups: w = dict(svg._data['virtual']) sofa_queries.append({'how': w['how'], 'with': w['with']}) k = (w['how'], w['with']) if k not in sofa_lut: sofa_lut[k] = [] sofa_lut[k].append(svg) # Check whether all year-group relations are in place # If there are two relations between an entity and a group in the # same year, we do not want this relation to be handled twice. # Thus we keep a seperate look-up-table to prevent this. year_vgroup_rel_ok = set() for mrel in year_group_mrels: g = groups[id2name[mrel['with']]] for year in mrel['years']: yg = groups[str(g.name) + str(year)] rrel = {'who': mrel['who'], 'with': yg._id, 'how': mrel['how'], 'from': DT_MIN, 'until': DT_MAX} if (not relkey(rrel) in vgroup_rlut and relkey(rrel) not in year_vgroup_rel_ok): logging.info("vgroup: adding %s -> %s (%s)" % ( id2name[mrel['who']], yg.name, id2name.get(mrel['how']))) Es.rcol.insert(rrel) elif relkey(rrel) in vgroup_rlut: del vgroup_rlut[relkey(rrel)] year_vgroup_rel_ok.add(relkey(rrel)) # Check whether all sofas are created sofa_brands = {} for b in Es.brands(): if tags['!sofa-brand'] not in b._data['tags']: continue sofa_brands[b._id] = b for rel in Es.query_relations(how=tuple(sofa_brands.values())): if (rel['how'], rel['with']) in sofa_lut: continue if not id2name[rel['with']]: continue g = groups[id2name[rel['with']]] nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix logging.info("creating sofa %s" % nm) n = {'types': ['group', 'tag'], 'names': [nm], 'tags': [tags['!virtual-group']], 'virtual': { 'type': 'sofa', 'with': rel['with'], 'how': rel['how']}, 'humanNames': [{ 'name': nm, 'human': six.text_type(g.humanName) + ' ' + six.text_type(sofa_brands[rel['how']].humanName)}]} n['_id'] = Es.ecol.insert(n) groups[nm] = Es.Group(n) id2name[n['_id']] = nm sofa_vgroups.append(g) sofa_lut[rel['how'], rel['with']] = [groups[nm]] sofa_queries.append({'how': rel['how'], 'with': rel['with']}) # Find all relations for the sofa virtual groups and check whether # the appropriate relations to the sofas are generated for rel in Es.disj_query_relations(sofa_queries): for svg in sofa_lut[(rel['how'], rel['with'])]: rrel = {'how': None, 'from': rel['from'], 'until': rel['until'], 'who': rel['who'], 'with': svg._id} if not relkey(rrel) in vgroup_rlut: logging.info("sofa: adding %s to %s" % ( id2name[rrel['who']], str(svg.name))) Es.rcol.insert(rrel) else: del vgroup_rlut[relkey(rrel)] # Check which relations to vgroups are unaccounted for and thus are to # be removed. for relkey in vgroup_rlut: logging.info("removing superfluous %s -> %s (%s)" % ( id2name[relkey[0]], id2name.get(relkey[2]), id2name.get(relkey[1]))) Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3], relkey[4]) # Set is_active on Users if and only if they are not in the `leden' group. # TODO We might optimize this by including it in a more generic process active_users = [rel['who'] for rel in Es.by_name('leden').get_rrelated( None, dt_now, dt_now, False, False, False)] for u in Es.users(): is_active = u._id in active_users if u.is_active == is_active: continue u._data['is_active'] = is_active u.save() logging.info( "%s user %s", ("activated" if is_active else "deactivated"), str(u.name) )
def update_db(giedo): dt_now = now() # Load tags # TODO cache this tags = Es.ids_by_names(('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand')) year_overrides = {} for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag): year_overrides[t._id] = (t._data['year-override']['type'], t._data['year-override']['year']) # Load _id -> name lut. id2name = Es.names_by_ids() # Load groups and brands groups = Es.of_type_by_name('group') groups_set = frozenset(groups.values()) # Find groups that have a virtual group for each year year_groups = [g for g in groups.itervalues() if tags['!year-group'] in g.tag_ids] # Find relations on those groups and add the years for which those # relations hold. def add_years_to_relations(rels): until_years = [Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX] max_until = max(max(until_years) if until_years else 0, Es.date_to_year(dt_now)) from_years = [Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN] min_from = min(min(from_years) if from_years else Es.date_to_year(dt_now), Es.date_to_year(dt_now)) for rel in rels: s = min_from if rel['from'] == DT_MIN \ else Es.date_to_year(rel['from']) t = max_until if rel['until'] == DT_MAX \ else Es.date_to_year(rel['until']) years = set(range(s, t+1)) for tid in rel.get('tags', ()): if tid not in year_overrides: continue yr, tp = year_overrides[tid] if tp: years.add(yr) else: years.remove(yr) rel['years'] = years year_group_mrels = tuple(Es.query_relations( _with=year_groups)) # Check whether all year groups are created for g in year_groups: mrels = filter(lambda x: x['with']==g._id, year_group_mrels) add_years_to_relations(mrels) years = set() for rel in mrels: years.update(rel['years']) for year in years: n = str(g.name) + str(year) if n not in groups: logging.info("Creating yeargroup %s" % n) _create_yeargroup(g, year, n, tags, groups, id2name) # Find all virtual groups virtual_groups = [g for g in groups_set if tags['!virtual-group'] in g.tag_ids] sofa_vgroups = [] yeargroup_vgroups = [] for vg in virtual_groups: if vg._data['virtual']['type'] == 'sofa': sofa_vgroups.append(vg) elif vg._data['virtual']['type'] == 'year-group': yeargroup_vgroups.append(vg) else: logging.warn("Unknown vgroup type: %s" \ % vg._data['virtua']['type']) # Find all relations with the sofa virtual groups def relkey(rel): return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until']) vgroup_rlut = dict() for rel in Es.query_relations(_with=virtual_groups): vgroup_rlut[relkey(rel)] = rel['_id'] # create look up table of existing sofas sofa_queries = [] sofa_lut = dict() for svg in sofa_vgroups: w = dict(svg._data['virtual']) sofa_queries.append({'how': w['how'], 'with': w['with']}) k = (w['how'], w['with']) if k not in sofa_lut: sofa_lut[k] = [] sofa_lut[k].append(svg) # Check whether all year-group relations are in place for mrel in year_group_mrels: g = groups[id2name[mrel['with']]] for year in mrel['years']: yg = groups[str(g.name) + str(year)] rrel = {'who': mrel['who'], 'with': yg._id, 'how': mrel['how'], 'from': DT_MIN, 'until': DT_MAX} if not relkey(rrel) in vgroup_rlut: logging.info("vgroup: adding %s -> %s (%s)" % ( id2name[mrel['who']], yg.name, id2name.get(mrel['how']))) Es.rcol.insert(rrel) else: del vgroup_rlut[relkey(rrel)] # Check whether all sofas are created sofa_brands = {} for b in Es.brands(): if tags['!sofa-brand'] not in b._data['tags']: continue sofa_brands[b._id] = b for rel in Es.query_relations(how=sofa_brands.values()): if (rel['how'], rel['with']) in sofa_lut: continue g = groups[id2name[rel['with']]] nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix logging.info("creating sofa %s" % nm) n = {'types': ['group','tag'], 'names': [nm], 'tags': [tags['!virtual-group']], 'virtual': { 'type': 'sofa', 'with': rel['with'], 'how': rel['how']}, 'humanNames': [{ 'name': nm, 'human': unicode(g.humanName) + ' ' + unicode(sofa_brands[rel['how']].humanName)}]} n['_id'] = Es.ecol.insert(n) groups[nm] = Es.Group(n) id2name[n['_id']] = nm sofa_vgroups.append(g) sofa_lut[rel['how'], rel['with']] = [groups[nm]] sofa_queries.append({'how': rel['how'], 'with': rel['with']}) # Find all relations for the sofa virtual groups and check whether # the appropriate relations to the sofas are generated for rel in Es.disj_query_relations(sofa_queries): for svg in sofa_lut[(rel['how'], rel['with'])]: rrel = {'how': None, 'from': rel['from'], 'until': rel['until'], 'who': rel['who'], 'with': svg._id} if not relkey(rrel) in vgroup_rlut: logging.info("sofa: adding %s to %s" % ( id2name[rrel['who']], str(svg.name))) Es.rcol.insert(rrel) else: del vgroup_rlut[relkey(rrel)] # Check which relations to vgroups are unaccounted for and thus are to # be removed. for relkey in vgroup_rlut: logging.info("removing superfluous %s -> %s (%s)" % ( id2name[relkey[0]], id2name.get(relkey[2]), id2name.get(relkey[1])))
def main(data): def str_to_date(s): if s is None: return None return datetime(*strptime(s, "%Y-%m-%d")[:3]) def year_to_dates(year): return (datetime(2003 + year, 9, 1), datetime(2004 + year, 8, 31)) def create_tag(name, humanName, tags=[]): return Es.ecol.insert( {"types": ["tag"], "names": [name], "humanNames": [{"name": name, "human": humanName}], "tags": tags} ) print "dropping" Es.ecol.drop() Es.rcol.drop() Es.mcol.drop() subscr_Es.ecol.drop() subscr_Es.scol.drop() print "creating indices" Es.ensure_indices() subscr_Es.ensure_indices() mod_Es.ensure_indices() conv_inst = dict() conv_study = dict() conv_group = dict() conv_group_byname = dict() conv_event = dict() conv_seat = dict() conv_user = dict() ignore_groups = frozenset(["leden-oud"]) ignore_groups_members = frozenset(["leden"]) ignore_groups_ids = set() ignore_groups_members_ids = set() year_groups = frozenset( ["leden" + str(x) for x in range(1, 9)] + ["kasco" + str(x) for x in range(1, 9)] + ["bestuur" + str(x) for x in range(1, 9)] ) year_groups_ids = dict() year_groups_lut = {} print "initial tags" system_tag = create_tag("!system", "Systeemstempels") year_overrides_tag = create_tag("!year-overrides", "Jaarlidmaatschapstempels", [system_tag]) virtual_group_tag = create_tag("!virtual-group", "Virtuele groep", [system_tag]) sofa_brand_tag = create_tag("!sofa-brand", "Sofa merk", [system_tag]) year_group_tag = create_tag("!year-group", "Jaargroep", [system_tag]) for i in xrange(1, 9): Es.ecol.insert( { "types": ["tag"], "humanNames": [{"human": "Wel jaar %s" % i}], "year-override": {"year": i, "type": True}, "tags": [year_overrides_tag], } ) Es.ecol.insert( { "types": ["tag"], "humanNames": [{"human": "Niet jaar %s" % i}], "year-override": {"year": i, "type": False}, "tags": [year_overrides_tag], } ) print "institutes" for m in data["EduInstitute"]: n = {"types": ["institute"], "humanNames": [{"human": m["name"]}]} conv_inst[m["id"]] = Es.ecol.insert(n) print "studies" for m in data["Study"]: n = {"types": ["study"], "humanNames": [{"human": m["name"]}]} conv_study[m["id"]] = Es.ecol.insert(n) print "initial groups" conv_group_byname["bestuur"] = { "id": Es.ecol.insert( { "types": ["group"], "names": ["bestuur"], "tags": [year_group_tag], "humanNames": [{"name": "bestuur", "human": "Bestuur", "genitive_prefix": "van het"}], "description": "Het bestuur", } ), "name": "bestuur", "humanName": "Bestuur", } conv_group_byname["kasco"] = { "id": Es.ecol.insert( { "types": ["group"], "names": ["kasco"], "tags": [year_group_tag], "humanNames": [{"name": "kasco", "human": "Kascontrolecommissie", "genitive_prefix": "van de"}], "description": "De kascontrolecommissie", } ), "name": "kasco", "humanName": "Bestuur", } print "groups" for m in data["OldKnGroup"]: if m["name"] in ignore_groups: ignore_groups_ids.add(m["id"]) continue if m["name"] in ignore_groups_members: ignore_groups_members_ids.add(m["id"]) if m["name"] in year_groups: year_groups_ids[m["id"]] = m["name"] group = m["name"][:-1] year = int(m["name"][-1:]) year_groups_lut[m["id"]] = (group, year) continue if m["name"] == "leden": m["isVirtual"] = False # fix for leden n = { "types": ["tag" if m["isVirtual"] else "group"], "names": [m["name"]], "humanNames": [{"name": m["name"], "human": m["humanName"], "genitive_prefix": m["genitive_prefix"]}], "description": m["description"], "temp": {"is_virtual": m["isVirtual"]}, } conv_group[m["id"]] = {"id": Es.ecol.insert(n), "name": m["name"], "humanName": m["humanName"]} conv_group_byname[m["name"]] = conv_group[m["id"]] print "group hierarchy" for m in data["OldKnGroup"]: if m["name"] in year_groups or m["name"] in ignore_groups: continue if m["parent"] is not None: if not m["parent"] in conv_group: print " %s was orphaned" % m["name"] continue Es.ecol.update({"_id": conv_group[m["id"]]["id"]}, {"$push": {"tags": conv_group[m["parent"]]["id"]}}) print "users" for m in data["OldKnUser"]: bits = m["password"].split("$") if len(bits) == 3: pwd = {"algorithm": bits[0], "salt": bits[1], "hash": bits[2]} else: pwd = None n = { "types": ["user"], "names": [m["username"]], "humanNames": [{"human": m["first_name"] + " " + m["last_name"]}], "person": { "titles": [], "nick": m["first_name"], "given": None, "family": m["last_name"], "gender": m["gender"], "dateOfBirth": str_to_date(m["dateOfBirth"]), }, "emailAddresses": [{"email": m["email"], "from": DT_MIN, "until": DT_MAX}], "addresses": [ { "street": m["addr_street"], "number": m["addr_number"], "zip": m["addr_zipCode"], "city": m["addr_city"], "from": DT_MIN, "until": DT_MAX, } ], "telephones": [{"number": m["telephone"], "from": DT_MIN, "until": DT_MAX}], "studies": [ { "institute": conv_inst.get(m["institute"]), "study": conv_study.get(m["study"]), "from": DT_MIN, "until": DT_MAX, "number": m["studentNumber"], } ], "temp": { "oud": m["in_oud"], "aan": m["in_aan"], "incasso": m["got_incasso"], "joined": m["dateJoined"], "remarks": m["remarks"], }, "is_active": m["is_active"], "password": pwd, } conv_user[m["id"]] = Es.ecol.insert(n) for g in m["groups"]: if g in ignore_groups_ids or g in ignore_groups_members_ids: continue if g in year_groups_ids: gname, year = year_groups_lut[g] f, u = year_to_dates(year) Es.rcol.insert( { "with": conv_group_byname[gname]["id"], "who": conv_user[m["id"]], "from": f, "until": u, "how": None, } ) continue Es.rcol.insert( {"with": conv_group[g]["id"], "who": conv_user[m["id"]], "from": DT_MIN, "until": DT_MAX, "how": None} ) print "brands" for m in data["OldSeat"]: if m["name"] == "deelhoofd": m["name"] = "graficideelhoofd" if m["name"] in conv_seat: continue n = { "types": ["brand"], "names": [m["name"]], "tags": [sofa_brand_tag], "humanNames": [{"name": m["name"], "human": m["humanName"]}], } conv_seat[m["name"]] = {"id": Es.ecol.insert(n)} print "seats" for m in data["OldSeat"]: if m["group"] in year_groups_ids: gname = year_groups_ids[m["group"]] gdat = conv_group_byname[gname[:-1]] _from, until = year_to_dates(int(gname[-1:])) else: gdat = conv_group[m["group"]] _from, until = DT_MIN, DT_MAX n = { "types": ["group"], "names": [gdat["name"] + "-" + m["name"]], "description": [m["description"]], "tags": [virtual_group_tag], "virtual": {"type": "sofa", "with": gdat["id"], "how": conv_seat[m["name"]]["id"]}, "humanNames": [{"name": gdat["name"] + "-" + m["name"], "human": m["humanName"] + " " + gdat["humanName"]}], } i = Es.ecol.insert(n) Es.rcol.insert( { "who": conv_user[m["user"]], "from": _from, "until": until, "how": conv_seat[m["name"]]["id"], "with": gdat["id"], } ) print "merging relations" print " list until" lut = dict() plan_changes = dict() plan_remove = set() for r in Es.rcol.find({"until": {"$lt": DT_MAX}}): lut[r["until"] + timedelta(1, 0), r["with"], r["how"], r["who"]] = r["_id"] print " crossreference from" for r in Es.rcol.find({"from": {"$gt": DT_MIN}}): n = (r["from"], r["with"], r["how"], r["who"]) if n not in lut: continue plan_changes[lut[n]] = (r["until"], r["_id"]) plan_remove.add(r["_id"]) print " transitive closure of plan" print "small final tweaks to groups" Es.ecol.update({"names": "leden"}, {"$push": {"tags": year_group_tag}}) done = False while not done: done = True for k, v in plan_changes.iteritems(): if v[1] in plan_changes: plan_changes[k] = plan_changes[v[1]] del plan_changes[v[1]] done = False break print " execute" for r in plan_remove: Es.rcol.remove({"_id": r}) for k, v in plan_changes.iteritems(): Es.rcol.update({"_id": k}, {"$set": {"until": v[0]}}) print "event" for m in data["Event"]: if m["owner"] not in conv_group: gname = year_groups_lut[m["owner"]][0] gid = conv_group_byname[gname]["id"] else: gid = conv_group[m["owner"]]["id"] conv_event[m["id"]] = subscr_Es.ecol.insert( { "mailBody": m["mailBody"], "humanName": m["humanName"], "description": m["description"], "cost": m["cost"], "is_open": m["is_open"], "owner": gid, "name": m["name"], } ) print "event subscriptions" for m in data["EventSubscription"]: subscr_Es.scol.insert( { "event": conv_event[m["event"]], "userNotes": m["userNotes"], "debit": m["debit"], "user": conv_user[m["user"]], } ) print "giedo updatedb" update_db(None) print "alias" print " ids_by_names" name2id = Es.ids_by_names() print " to graph" alias_graph = {} for m in data["Alias"]: alias_graph[m["source"]] = m["target"] print " tarjan" for scc in tarjan(alias_graph): assert len(scc) == 1 src = scc[0] if src in name2id: continue if not src in alias_graph: continue if not alias_graph[src] in name2id: print " ? %s -> %s" % (src, alias_graph[src]) continue name2id[src] = name2id[alias_graph[src]] Es.ecol.update({"names": alias_graph[src]}, {"$push": {"names": src}})