def set_tags(self, tags, save=True):
'''
Set tags by their usernames.
'''
name2id = Es.ids_by_names(tags)
self._data['tags'] = []
for name in tags:
self._data['tags'].append(name2id[name])
if save:
self.save()
def set_tags(self, tags, save=True):
"""
Set tags by their usernames.
"""
name2id = Es.ids_by_names(tags)
self._data["tags"] = []
for name in tags:
self._data["tags"].append(name2id[name])
self.update_search_text(save=False)
if save:
self.save()
def set_tags(self, tags, save=True):
'''
Set tags by their usernames.
'''
name2id = Es.ids_by_names(tags)
self._data['tags'] = []
for name in tags:
self._data['tags'].append(name2id[name])
self.update_search_text(save=False)
if save:
self.save()
def main(data):
def str_to_date(s):
if s is None:
return None
return datetime(*strptime(s,'%Y-%m-%d')[:3])
def year_to_dates(year):
return (datetime(2003+year,9,1),
datetime(2004+year,8,31))
def create_tag(name, humanName, tags=[]):
return Es.ecol.insert({'types': ['tag'],
'names': [name],
'humanNames': [{
'name': name,
'human': humanName}],
'tags': tags})
print 'dropping'
Es.ecol.drop()
Es.rcol.drop()
Es.mcol.drop()
subscr_Es.ecol.drop()
subscr_Es.scol.drop()
print 'creating indices'
Es.ensure_indices()
subscr_Es.ensure_indices()
mod_Es.ensure_indices()
conv_inst = dict()
conv_study = dict()
conv_group = dict()
conv_group_byname = dict()
conv_event = dict()
conv_seat = dict()
conv_user = dict()
ignore_groups = frozenset(['leden-oud'])
ignore_groups_members = frozenset(['leden'])
ignore_groups_ids = set()
ignore_groups_members_ids = set()
year_groups = frozenset(
['leden'+str(x) for x in range(1,9)]+
['kasco'+str(x) for x in range(1,9)]+
['bestuur'+str(x) for x in range(1,9)])
year_groups_ids = dict()
year_groups_lut = {}
print 'initial tags'
system_tag = create_tag('!system', 'Systeemstempels')
year_overrides_tag = create_tag('!year-overrides',
'Jaarlidmaatschapstempels', [system_tag])
virtual_group_tag = create_tag("!virtual-group",
'Virtuele groep', [system_tag])
sofa_brand_tag = create_tag("!sofa-brand",
'Sofa merk', [system_tag])
year_group_tag = create_tag("!year-group", 'Jaargroep',
[system_tag])
for i in xrange(1,9):
Es.ecol.insert({'types': ['tag'],
'humanNames': [{'human': 'Wel jaar %s' % i}],
'year-override': {'year': i,
'type': True},
'tags': [year_overrides_tag]})
Es.ecol.insert({'types': ['tag'],
'humanNames': [{'human': 'Niet jaar %s' % i}],
'year-override': {'year': i,
'type': False},
'tags': [year_overrides_tag]})
print 'institutes'
for m in data['EduInstitute']:
n = { 'types': ['institute'],
'humanNames': [{'human': m['name']}]}
conv_inst[m['id']] = Es.ecol.insert(n)
print 'studies'
for m in data['Study']:
n = { 'types': ['study'],
'humanNames': [{'human': m['name']}]}
conv_study[m['id']] = Es.ecol.insert(n)
print 'initial groups'
conv_group_byname['bestuur'] = {
'id': Es.ecol.insert({
'types': ['group'],
'names': ['bestuur'],
'tags': [year_group_tag],
'humanNames': [{
'name': 'bestuur',
'human': 'Bestuur',
'genitive_prefix': 'van het'}],
'description': "Het bestuur"}),
'name': 'bestuur',
'humanName': 'Bestuur'}
conv_group_byname['kasco'] = {
'id': Es.ecol.insert({
'types': ['group'],
'names': ['kasco'],
'tags': [year_group_tag],
'humanNames': [{
'name': 'kasco',
'human': 'Kascontrolecommissie',
'genitive_prefix': 'van de'}],
'description': "De kascontrolecommissie"}),
'name': 'kasco',
'humanName': 'Bestuur'}
print 'groups'
for m in data['OldKnGroup']:
if m['name'] in ignore_groups:
ignore_groups_ids.add(m['id'])
continue
if m['name'] in ignore_groups_members:
ignore_groups_members_ids.add(m['id'])
if m['name'] in year_groups:
year_groups_ids[m['id']] = m['name']
group = m['name'][:-1]
year = int(m['name'][-1:])
year_groups_lut[m['id']] = (group, year)
continue
if m['name'] == 'leden': m['isVirtual'] = False # fix for leden
n = { 'types': ['tag' if m['isVirtual'] else 'group'],
'names': [m['name']],
'humanNames': [{
'name': m['name'],
'human': m['humanName'],
'genitive_prefix': m['genitive_prefix']
}],
'description': m['description'],
'temp':{
'is_virtual': m['isVirtual']
}
}
conv_group[m['id']] = {'id': Es.ecol.insert(n),
'name': m['name'],
'humanName': m['humanName']}
conv_group_byname[m['name']] = conv_group[m['id']]
print 'group hierarchy'
for m in data['OldKnGroup']:
if m['name'] in year_groups or m['name'] in ignore_groups:
continue
if m['parent'] is not None:
if not m['parent'] in conv_group:
print " %s was orphaned" % m['name']
continue
Es.ecol.update({'_id': conv_group[m['id']]['id']},
{'$push': {'tags': conv_group[
m['parent']]['id']}})
print 'users'
for m in data['OldKnUser']:
bits = m['password'].split('$')
if len(bits) == 3:
pwd = {'algorithm': bits[0],
'salt': bits[1],
'hash': bits[2]}
else:
pwd = None
n = {
'types': ['user'],
'names': [m['username']],
'humanNames': [{'human': m['first_name'] + ' ' +
m['last_name']}],
'person': {
'titles': [],
'nick': m['first_name'],
'given': None,
'family': m['last_name'],
'gender': m['gender'],
'dateOfBirth': str_to_date(m['dateOfBirth'])
},
'emailAddresses': [
{'email': m['email'],
'from': DT_MIN,
'until': DT_MAX
}],
'addresses': [
{'street': m['addr_street'],
'number': m['addr_number'],
'zip': m['addr_zipCode'],
'city': m['addr_city'],
'from': DT_MIN,
'until': DT_MAX
}],
'telephones': [
{'number': m['telephone'],
'from': DT_MIN,
'until': DT_MAX}],
'studies': [
{'institute': conv_inst.get(m['institute']),
'study': conv_study.get(m['study']),
'from': DT_MIN,
'until': DT_MAX,
'number': m['studentNumber']}
],
'temp': {
'oud': m['in_oud'],
'aan': m['in_aan'],
'incasso': m['got_incasso'],
'joined': m['dateJoined'],
'remarks': m['remarks']
},
'is_active': m['is_active'],
'password': pwd
}
conv_user[m['id']] = Es.ecol.insert(n)
for g in m['groups']:
if g in ignore_groups_ids or \
g in ignore_groups_members_ids:
continue
if g in year_groups_ids:
gname, year = year_groups_lut[g]
f, u = year_to_dates(year)
Es.rcol.insert({
'with': conv_group_byname[gname]['id'],
'who': conv_user[m['id']],
'from': f,
'until': u,
'how': None})
continue
Es.rcol.insert({
'with': conv_group[g]['id'],
'who': conv_user[m['id']],
'from': DT_MIN,
'until': DT_MAX,
'how': None})
print 'brands'
for m in data['OldSeat']:
if m['name'] == 'deelhoofd':
m['name'] = 'graficideelhoofd'
if m['name'] in conv_seat:
continue
n = {'types': ['brand'],
'names': [m['name']],
'tags': [sofa_brand_tag],
'humanNames': [{'name': m['name'],
'human': m['humanName']}]}
conv_seat[m['name']] = {'id': Es.ecol.insert(n)}
print 'seats'
for m in data['OldSeat']:
if m['group'] in year_groups_ids:
gname = year_groups_ids[m['group']]
gdat = conv_group_byname[gname[:-1]]
_from, until = year_to_dates(int(gname[-1:]))
else:
gdat = conv_group[m['group']]
_from, until = DT_MIN, DT_MAX
n = {'types': ['group'],
'names': [gdat['name'] + '-' + m['name']],
'description': [m['description']],
'tags': [virtual_group_tag],
'virtual': {
'type': 'sofa',
'with': gdat['id'],
'how': conv_seat[m['name']]['id']},
'humanNames': [{
'name': gdat['name'] + '-' + m['name'],
'human': m['humanName'] +
' ' + gdat['humanName']}]}
i = Es.ecol.insert(n)
Es.rcol.insert({'who': conv_user[m['user']],
'from': _from,
'until': until,
'how': conv_seat[m['name']]['id'],
'with': gdat['id']})
print 'merging relations'
print ' list until'
lut = dict()
plan_changes = dict()
plan_remove = set()
for r in Es.rcol.find({'until': {'$lt': DT_MAX}}):
lut[r['until'] + timedelta(1,0), r['with'],
r['how'], r['who']] = r['_id']
print ' crossreference from'
for r in Es.rcol.find({'from': {'$gt': DT_MIN}}):
n = (r['from'], r['with'], r['how'], r['who'])
if n not in lut:
continue
plan_changes[lut[n]] = (r['until'], r['_id'])
plan_remove.add(r['_id'])
print ' transitive closure of plan'
print 'small final tweaks to groups'
Es.ecol.update({'names': 'leden'}, {'$push': {'tags': year_group_tag}})
done = False
while not done:
done = True
for k,v in plan_changes.iteritems():
if v[1] in plan_changes:
plan_changes[k] = plan_changes[v[1]]
del plan_changes[v[1]]
done = False
break
print ' execute'
for r in plan_remove:
Es.rcol.remove({'_id': r})
for k,v in plan_changes.iteritems():
Es.rcol.update({'_id': k}, {'$set': {'until': v[0]}})
print 'event'
for m in data['Event']:
if m['owner'] not in conv_group:
gname = year_groups_lut[m['owner']][0]
gid = conv_group_byname[gname]['id']
else:
gid = conv_group[m['owner']]['id']
conv_event[m['id']] = subscr_Es.ecol.insert({
'mailBody': m['mailBody'],
'humanName': m['humanName'],
'description': m['description'],
'cost': m['cost'],
'is_open': m['is_open'],
'owner': gid,
'name': m['name']})
print 'event subscriptions'
for m in data['EventSubscription']:
subscr_Es.scol.insert({
'event': conv_event[m['event']],
'userNotes': m['userNotes'],
'debit': m['debit'],
'user': conv_user[m['user']]})
print 'giedo updatedb'
update_db(None)
print 'alias'
print ' ids_by_names'
name2id = Es.ids_by_names()
print ' to graph'
alias_graph = {}
for m in data['Alias']:
alias_graph[m['source']] = m['target']
print ' tarjan'
for scc in tarjan(alias_graph):
assert len(scc) == 1
src = scc[0]
if src in name2id:
continue
if not src in alias_graph:
continue
if not alias_graph[src] in name2id:
print ' ? %s -> %s' % (src, alias_graph[src])
continue
name2id[src] = name2id[alias_graph[src]]
Es.ecol.update({'names': alias_graph[src]},
{'$push': {'names': src}})
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(
('!year-group', '!year-overrides', '!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(six.itervalues(groups))
# Find groups that have a virtual group for each year
year_groups = [g for g in groups_set if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
years_of_year_overrides = [
yo[1] for yo in six.itervalues(year_overrides)
]
until_years = [
Es.date_to_year(r['until']) for r in rels if r['until'] != DT_MAX
]
max_until = max(
max(until_years) if until_years else 0,
max(years_of_year_overrides) if years_of_year_overrides else 0,
Es.date_to_year(dt_now))
from_years = [
Es.date_to_year(r['from']) for r in rels if r['from'] != DT_MIN
]
min_from = min(
min(from_years) if from_years else Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = (min_from
if rel['from'] == DT_MIN else Es.date_to_year(rel['from']))
t = (max_until
if rel['until'] == DT_MAX else Es.date_to_year(rel['until']))
years = set(range(s, t + 1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
tp, yr = year_overrides[tid]
if tp:
years.add(yr)
else:
if yr not in years:
logging.warning('bogus year-override -{} on {}'.format(
yr, rel['_id']))
continue
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups, id2name)
# Find all virtual groups
virtual_groups = [
g for g in groups_set if tags['!virtual-group'] in g.tag_ids
]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s" %
vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'], rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'], 'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
# If there are two relations between an entity and a group in the
# same year, we do not want this relation to be handled twice.
# Thus we keep a seperate look-up-table to prevent this.
year_vgroup_rel_ok = set()
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {
'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX
}
if (not relkey(rrel) in vgroup_rlut
and relkey(rrel) not in year_vgroup_rel_ok):
logging.info(
"vgroup: adding %s -> %s (%s)" %
(id2name[mrel['who']], yg.name, id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
elif relkey(rrel) in vgroup_rlut:
del vgroup_rlut[relkey(rrel)]
year_vgroup_rel_ok.add(relkey(rrel))
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=tuple(sofa_brands.values())):
if (rel['how'], rel['with']) in sofa_lut:
continue
if not id2name[rel['with']]:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {
'types': ['group', 'tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']
},
'humanNames': [{
'name':
nm,
'human':
six.text_type(g.humanName) + ' ' +
six.text_type(sofa_brands[rel['how']].humanName)
}]
}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'], 'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {
'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id
}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" %
(id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" %
(id2name[relkey[0]], id2name.get(
relkey[2]), id2name.get(relkey[1])))
Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
relkey[4])
# Set is_active on Users if and only if they are not in the `leden' group.
# TODO We might optimize this by including it in a more generic process
active_users = [
rel['who'] for rel in Es.by_name('leden').get_rrelated(
None, dt_now, dt_now, False, False, False)
]
for u in Es.users():
is_active = u._id in active_users
if u.is_active == is_active:
continue
u._data['is_active'] = is_active
u.save()
logging.info("%s user %s",
("activated" if is_active else "deactivated"),
str(u.name))
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(('!year-group', '!year-overrides',
'!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(six.itervalues(groups))
# Find groups that have a virtual group for each year
year_groups = [g for g in groups_set
if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
years_of_year_overrides = [yo[1] for yo
in six.itervalues(year_overrides)]
until_years = [Es.date_to_year(r['until']) for r in rels
if r['until'] != DT_MAX]
max_until = max(
max(until_years) if until_years else 0,
max(years_of_year_overrides) if years_of_year_overrides else 0,
Es.date_to_year(dt_now)
)
from_years = [Es.date_to_year(r['from']) for r in rels
if r['from'] != DT_MIN]
min_from = min(min(from_years) if from_years else
Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = (min_from if rel['from'] == DT_MIN
else Es.date_to_year(rel['from']))
t = (max_until if rel['until'] == DT_MAX
else Es.date_to_year(rel['until']))
years = set(range(s, t + 1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
tp, yr = year_overrides[tid]
if tp:
years.add(yr)
else:
if yr not in years:
logging.warning('bogus year-override -{} on {}'.format(
yr, rel['_id']))
continue
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = [rel for rel in year_group_mrels if rel['with'] == g._id]
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups, id2name)
# Find all virtual groups
virtual_groups = [g for g in groups_set
if tags['!virtual-group'] in g.tag_ids]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s"
% vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'],
rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'],
'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
# If there are two relations between an entity and a group in the
# same year, we do not want this relation to be handled twice.
# Thus we keep a seperate look-up-table to prevent this.
year_vgroup_rel_ok = set()
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX}
if (not relkey(rrel) in vgroup_rlut and relkey(rrel)
not in year_vgroup_rel_ok):
logging.info("vgroup: adding %s -> %s (%s)" % (
id2name[mrel['who']], yg.name,
id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
elif relkey(rrel) in vgroup_rlut:
del vgroup_rlut[relkey(rrel)]
year_vgroup_rel_ok.add(relkey(rrel))
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=tuple(sofa_brands.values())):
if (rel['how'], rel['with']) in sofa_lut:
continue
if not id2name[rel['with']]:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {'types': ['group', 'tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']},
'humanNames': [{
'name': nm,
'human': six.text_type(g.humanName) + ' ' +
six.text_type(sofa_brands[rel['how']].humanName)}]}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'],
'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" % (
id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" % (
id2name[relkey[0]], id2name.get(relkey[2]),
id2name.get(relkey[1])))
Es.remove_relation(relkey[0], relkey[2], relkey[1], relkey[3],
relkey[4])
# Set is_active on Users if and only if they are not in the `leden' group.
# TODO We might optimize this by including it in a more generic process
active_users = [rel['who'] for rel in Es.by_name('leden').get_rrelated(
None, dt_now, dt_now, False, False, False)]
for u in Es.users():
is_active = u._id in active_users
if u.is_active == is_active:
continue
u._data['is_active'] = is_active
u.save()
logging.info(
"%s user %s",
("activated" if is_active else "deactivated"),
str(u.name)
)
def update_db(giedo):
dt_now = now()
# Load tags
# TODO cache this
tags = Es.ids_by_names(('!year-group', '!year-overrides',
'!virtual-group', '!sofa-brand'))
year_overrides = {}
for t in Es.bearers_by_tag_id(tags['!year-overrides'], _as=Es.Tag):
year_overrides[t._id] = (t._data['year-override']['type'],
t._data['year-override']['year'])
# Load _id -> name lut.
id2name = Es.names_by_ids()
# Load groups and brands
groups = Es.of_type_by_name('group')
groups_set = frozenset(groups.values())
# Find groups that have a virtual group for each year
year_groups = [g for g in groups.itervalues()
if tags['!year-group'] in g.tag_ids]
# Find relations on those groups and add the years for which those
# relations hold.
def add_years_to_relations(rels):
until_years = [Es.date_to_year(r['until']) for r in rels
if r['until'] != DT_MAX]
max_until = max(max(until_years) if until_years else 0,
Es.date_to_year(dt_now))
from_years = [Es.date_to_year(r['from']) for r in rels
if r['from'] != DT_MIN]
min_from = min(min(from_years) if from_years else
Es.date_to_year(dt_now),
Es.date_to_year(dt_now))
for rel in rels:
s = min_from if rel['from'] == DT_MIN \
else Es.date_to_year(rel['from'])
t = max_until if rel['until'] == DT_MAX \
else Es.date_to_year(rel['until'])
years = set(range(s, t+1))
for tid in rel.get('tags', ()):
if tid not in year_overrides:
continue
yr, tp = year_overrides[tid]
if tp:
years.add(yr)
else:
years.remove(yr)
rel['years'] = years
year_group_mrels = tuple(Es.query_relations(
_with=year_groups))
# Check whether all year groups are created
for g in year_groups:
mrels = filter(lambda x: x['with']==g._id, year_group_mrels)
add_years_to_relations(mrels)
years = set()
for rel in mrels:
years.update(rel['years'])
for year in years:
n = str(g.name) + str(year)
if n not in groups:
logging.info("Creating yeargroup %s" % n)
_create_yeargroup(g, year, n, tags, groups,
id2name)
# Find all virtual groups
virtual_groups = [g for g in groups_set
if tags['!virtual-group'] in g.tag_ids]
sofa_vgroups = []
yeargroup_vgroups = []
for vg in virtual_groups:
if vg._data['virtual']['type'] == 'sofa':
sofa_vgroups.append(vg)
elif vg._data['virtual']['type'] == 'year-group':
yeargroup_vgroups.append(vg)
else:
logging.warn("Unknown vgroup type: %s" \
% vg._data['virtua']['type'])
# Find all relations with the sofa virtual groups
def relkey(rel):
return (rel['who'], rel['how'], rel['with'],
rel['from'], rel['until'])
vgroup_rlut = dict()
for rel in Es.query_relations(_with=virtual_groups):
vgroup_rlut[relkey(rel)] = rel['_id']
# create look up table of existing sofas
sofa_queries = []
sofa_lut = dict()
for svg in sofa_vgroups:
w = dict(svg._data['virtual'])
sofa_queries.append({'how': w['how'],
'with': w['with']})
k = (w['how'], w['with'])
if k not in sofa_lut:
sofa_lut[k] = []
sofa_lut[k].append(svg)
# Check whether all year-group relations are in place
for mrel in year_group_mrels:
g = groups[id2name[mrel['with']]]
for year in mrel['years']:
yg = groups[str(g.name) + str(year)]
rrel = {'who': mrel['who'],
'with': yg._id,
'how': mrel['how'],
'from': DT_MIN,
'until': DT_MAX}
if not relkey(rrel) in vgroup_rlut:
logging.info("vgroup: adding %s -> %s (%s)" % (
id2name[mrel['who']], yg.name,
id2name.get(mrel['how'])))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check whether all sofas are created
sofa_brands = {}
for b in Es.brands():
if tags['!sofa-brand'] not in b._data['tags']:
continue
sofa_brands[b._id] = b
for rel in Es.query_relations(how=sofa_brands.values()):
if (rel['how'], rel['with']) in sofa_lut:
continue
g = groups[id2name[rel['with']]]
nm = str(g.name) + '-' + sofa_brands[rel['how']].sofa_suffix
logging.info("creating sofa %s" % nm)
n = {'types': ['group','tag'],
'names': [nm],
'tags': [tags['!virtual-group']],
'virtual': {
'type': 'sofa',
'with': rel['with'],
'how': rel['how']},
'humanNames': [{
'name': nm,
'human': unicode(g.humanName) + ' ' +
unicode(sofa_brands[rel['how']].humanName)}]}
n['_id'] = Es.ecol.insert(n)
groups[nm] = Es.Group(n)
id2name[n['_id']] = nm
sofa_vgroups.append(g)
sofa_lut[rel['how'], rel['with']] = [groups[nm]]
sofa_queries.append({'how': rel['how'],
'with': rel['with']})
# Find all relations for the sofa virtual groups and check whether
# the appropriate relations to the sofas are generated
for rel in Es.disj_query_relations(sofa_queries):
for svg in sofa_lut[(rel['how'], rel['with'])]:
rrel = {'how': None,
'from': rel['from'],
'until': rel['until'],
'who': rel['who'],
'with': svg._id}
if not relkey(rrel) in vgroup_rlut:
logging.info("sofa: adding %s to %s" % (
id2name[rrel['who']], str(svg.name)))
Es.rcol.insert(rrel)
else:
del vgroup_rlut[relkey(rrel)]
# Check which relations to vgroups are unaccounted for and thus are to
# be removed.
for relkey in vgroup_rlut:
logging.info("removing superfluous %s -> %s (%s)" % (
id2name[relkey[0]], id2name.get(relkey[2]),
id2name.get(relkey[1])))
def main(data):
def str_to_date(s):
if s is None:
return None
return datetime(*strptime(s, "%Y-%m-%d")[:3])
def year_to_dates(year):
return (datetime(2003 + year, 9, 1), datetime(2004 + year, 8, 31))
def create_tag(name, humanName, tags=[]):
return Es.ecol.insert(
{"types": ["tag"], "names": [name], "humanNames": [{"name": name, "human": humanName}], "tags": tags}
)
print "dropping"
Es.ecol.drop()
Es.rcol.drop()
Es.mcol.drop()
subscr_Es.ecol.drop()
subscr_Es.scol.drop()
print "creating indices"
Es.ensure_indices()
subscr_Es.ensure_indices()
mod_Es.ensure_indices()
conv_inst = dict()
conv_study = dict()
conv_group = dict()
conv_group_byname = dict()
conv_event = dict()
conv_seat = dict()
conv_user = dict()
ignore_groups = frozenset(["leden-oud"])
ignore_groups_members = frozenset(["leden"])
ignore_groups_ids = set()
ignore_groups_members_ids = set()
year_groups = frozenset(
["leden" + str(x) for x in range(1, 9)]
+ ["kasco" + str(x) for x in range(1, 9)]
+ ["bestuur" + str(x) for x in range(1, 9)]
)
year_groups_ids = dict()
year_groups_lut = {}
print "initial tags"
system_tag = create_tag("!system", "Systeemstempels")
year_overrides_tag = create_tag("!year-overrides", "Jaarlidmaatschapstempels", [system_tag])
virtual_group_tag = create_tag("!virtual-group", "Virtuele groep", [system_tag])
sofa_brand_tag = create_tag("!sofa-brand", "Sofa merk", [system_tag])
year_group_tag = create_tag("!year-group", "Jaargroep", [system_tag])
for i in xrange(1, 9):
Es.ecol.insert(
{
"types": ["tag"],
"humanNames": [{"human": "Wel jaar %s" % i}],
"year-override": {"year": i, "type": True},
"tags": [year_overrides_tag],
}
)
Es.ecol.insert(
{
"types": ["tag"],
"humanNames": [{"human": "Niet jaar %s" % i}],
"year-override": {"year": i, "type": False},
"tags": [year_overrides_tag],
}
)
print "institutes"
for m in data["EduInstitute"]:
n = {"types": ["institute"], "humanNames": [{"human": m["name"]}]}
conv_inst[m["id"]] = Es.ecol.insert(n)
print "studies"
for m in data["Study"]:
n = {"types": ["study"], "humanNames": [{"human": m["name"]}]}
conv_study[m["id"]] = Es.ecol.insert(n)
print "initial groups"
conv_group_byname["bestuur"] = {
"id": Es.ecol.insert(
{
"types": ["group"],
"names": ["bestuur"],
"tags": [year_group_tag],
"humanNames": [{"name": "bestuur", "human": "Bestuur", "genitive_prefix": "van het"}],
"description": "Het bestuur",
}
),
"name": "bestuur",
"humanName": "Bestuur",
}
conv_group_byname["kasco"] = {
"id": Es.ecol.insert(
{
"types": ["group"],
"names": ["kasco"],
"tags": [year_group_tag],
"humanNames": [{"name": "kasco", "human": "Kascontrolecommissie", "genitive_prefix": "van de"}],
"description": "De kascontrolecommissie",
}
),
"name": "kasco",
"humanName": "Bestuur",
}
print "groups"
for m in data["OldKnGroup"]:
if m["name"] in ignore_groups:
ignore_groups_ids.add(m["id"])
continue
if m["name"] in ignore_groups_members:
ignore_groups_members_ids.add(m["id"])
if m["name"] in year_groups:
year_groups_ids[m["id"]] = m["name"]
group = m["name"][:-1]
year = int(m["name"][-1:])
year_groups_lut[m["id"]] = (group, year)
continue
if m["name"] == "leden":
m["isVirtual"] = False # fix for leden
n = {
"types": ["tag" if m["isVirtual"] else "group"],
"names": [m["name"]],
"humanNames": [{"name": m["name"], "human": m["humanName"], "genitive_prefix": m["genitive_prefix"]}],
"description": m["description"],
"temp": {"is_virtual": m["isVirtual"]},
}
conv_group[m["id"]] = {"id": Es.ecol.insert(n), "name": m["name"], "humanName": m["humanName"]}
conv_group_byname[m["name"]] = conv_group[m["id"]]
print "group hierarchy"
for m in data["OldKnGroup"]:
if m["name"] in year_groups or m["name"] in ignore_groups:
continue
if m["parent"] is not None:
if not m["parent"] in conv_group:
print " %s was orphaned" % m["name"]
continue
Es.ecol.update({"_id": conv_group[m["id"]]["id"]}, {"$push": {"tags": conv_group[m["parent"]]["id"]}})
print "users"
for m in data["OldKnUser"]:
bits = m["password"].split("$")
if len(bits) == 3:
pwd = {"algorithm": bits[0], "salt": bits[1], "hash": bits[2]}
else:
pwd = None
n = {
"types": ["user"],
"names": [m["username"]],
"humanNames": [{"human": m["first_name"] + " " + m["last_name"]}],
"person": {
"titles": [],
"nick": m["first_name"],
"given": None,
"family": m["last_name"],
"gender": m["gender"],
"dateOfBirth": str_to_date(m["dateOfBirth"]),
},
"emailAddresses": [{"email": m["email"], "from": DT_MIN, "until": DT_MAX}],
"addresses": [
{
"street": m["addr_street"],
"number": m["addr_number"],
"zip": m["addr_zipCode"],
"city": m["addr_city"],
"from": DT_MIN,
"until": DT_MAX,
}
],
"telephones": [{"number": m["telephone"], "from": DT_MIN, "until": DT_MAX}],
"studies": [
{
"institute": conv_inst.get(m["institute"]),
"study": conv_study.get(m["study"]),
"from": DT_MIN,
"until": DT_MAX,
"number": m["studentNumber"],
}
],
"temp": {
"oud": m["in_oud"],
"aan": m["in_aan"],
"incasso": m["got_incasso"],
"joined": m["dateJoined"],
"remarks": m["remarks"],
},
"is_active": m["is_active"],
"password": pwd,
}
conv_user[m["id"]] = Es.ecol.insert(n)
for g in m["groups"]:
if g in ignore_groups_ids or g in ignore_groups_members_ids:
continue
if g in year_groups_ids:
gname, year = year_groups_lut[g]
f, u = year_to_dates(year)
Es.rcol.insert(
{
"with": conv_group_byname[gname]["id"],
"who": conv_user[m["id"]],
"from": f,
"until": u,
"how": None,
}
)
continue
Es.rcol.insert(
{"with": conv_group[g]["id"], "who": conv_user[m["id"]], "from": DT_MIN, "until": DT_MAX, "how": None}
)
print "brands"
for m in data["OldSeat"]:
if m["name"] == "deelhoofd":
m["name"] = "graficideelhoofd"
if m["name"] in conv_seat:
continue
n = {
"types": ["brand"],
"names": [m["name"]],
"tags": [sofa_brand_tag],
"humanNames": [{"name": m["name"], "human": m["humanName"]}],
}
conv_seat[m["name"]] = {"id": Es.ecol.insert(n)}
print "seats"
for m in data["OldSeat"]:
if m["group"] in year_groups_ids:
gname = year_groups_ids[m["group"]]
gdat = conv_group_byname[gname[:-1]]
_from, until = year_to_dates(int(gname[-1:]))
else:
gdat = conv_group[m["group"]]
_from, until = DT_MIN, DT_MAX
n = {
"types": ["group"],
"names": [gdat["name"] + "-" + m["name"]],
"description": [m["description"]],
"tags": [virtual_group_tag],
"virtual": {"type": "sofa", "with": gdat["id"], "how": conv_seat[m["name"]]["id"]},
"humanNames": [{"name": gdat["name"] + "-" + m["name"], "human": m["humanName"] + " " + gdat["humanName"]}],
}
i = Es.ecol.insert(n)
Es.rcol.insert(
{
"who": conv_user[m["user"]],
"from": _from,
"until": until,
"how": conv_seat[m["name"]]["id"],
"with": gdat["id"],
}
)
print "merging relations"
print " list until"
lut = dict()
plan_changes = dict()
plan_remove = set()
for r in Es.rcol.find({"until": {"$lt": DT_MAX}}):
lut[r["until"] + timedelta(1, 0), r["with"], r["how"], r["who"]] = r["_id"]
print " crossreference from"
for r in Es.rcol.find({"from": {"$gt": DT_MIN}}):
n = (r["from"], r["with"], r["how"], r["who"])
if n not in lut:
continue
plan_changes[lut[n]] = (r["until"], r["_id"])
plan_remove.add(r["_id"])
print " transitive closure of plan"
print "small final tweaks to groups"
Es.ecol.update({"names": "leden"}, {"$push": {"tags": year_group_tag}})
done = False
while not done:
done = True
for k, v in plan_changes.iteritems():
if v[1] in plan_changes:
plan_changes[k] = plan_changes[v[1]]
del plan_changes[v[1]]
done = False
break
print " execute"
for r in plan_remove:
Es.rcol.remove({"_id": r})
for k, v in plan_changes.iteritems():
Es.rcol.update({"_id": k}, {"$set": {"until": v[0]}})
print "event"
for m in data["Event"]:
if m["owner"] not in conv_group:
gname = year_groups_lut[m["owner"]][0]
gid = conv_group_byname[gname]["id"]
else:
gid = conv_group[m["owner"]]["id"]
conv_event[m["id"]] = subscr_Es.ecol.insert(
{
"mailBody": m["mailBody"],
"humanName": m["humanName"],
"description": m["description"],
"cost": m["cost"],
"is_open": m["is_open"],
"owner": gid,
"name": m["name"],
}
)
print "event subscriptions"
for m in data["EventSubscription"]:
subscr_Es.scol.insert(
{
"event": conv_event[m["event"]],
"userNotes": m["userNotes"],
"debit": m["debit"],
"user": conv_user[m["user"]],
}
)
print "giedo updatedb"
update_db(None)
print "alias"
print " ids_by_names"
name2id = Es.ids_by_names()
print " to graph"
alias_graph = {}
for m in data["Alias"]:
alias_graph[m["source"]] = m["target"]
print " tarjan"
for scc in tarjan(alias_graph):
assert len(scc) == 1
src = scc[0]
if src in name2id:
continue
if not src in alias_graph:
continue
if not alias_graph[src] in name2id:
print " ? %s -> %s" % (src, alias_graph[src])
continue
name2id[src] = name2id[alias_graph[src]]
Es.ecol.update({"names": alias_graph[src]}, {"$push": {"names": src}})
