Exemple #1
0
def main():
    print 'Are you sure you want to run this?'
    return
    db.command('dropDatabase')
    print 'Creating indices'
    print ' leden'
    Es.ensure_indices()
    print ' moderation'
    Es_mod.ensure_indices()
    print ' planning'
    Es_plan.ensure_indices()
    print ' poll'
    Es_poll.ensure_indices()
    print ' regl'
    Es_regl.ensure_indices()
    print ' subscriptions'
    Es_subscr.ensure_indices()
    print
    print 'Restoring data'
    print ' entities'
    for e in bson.decode_all(open('entities.bsons').read()):
        db['entities'].save(e)
    print ' relations'
    for e in bson.decode_all(open('relations.bsons').read()):
        db['relations'].save(e)
    print ' events'
    for e in bson.decode_all(open('events.bsons').read()):
        db['events'].save(e)
Exemple #2
0
def main():
    print 'Are you sure you want to run this?'
    return
    db.command('dropDatabase')
    print 'Creating indices'
    print ' leden'
    Es.ensure_indices()
    print ' moderation'
    Es_mod.ensure_indices()
    print ' planning'
    Es_plan.ensure_indices()
    print ' poll'
    Es_poll.ensure_indices()
    print ' regl'
    Es_regl.ensure_indices()
    print ' subscriptions'
    Es_subscr.ensure_indices()
    print
    print 'Restoring data'
    print ' entities'
    for e in bson.decode_all(open('entities.bsons').read()):
        db['entities'].save(e)
    print ' relations'
    for e in bson.decode_all(open('relations.bsons').read()):
        db['relations'].save(e)
    print ' events'
    for e in bson.decode_all(open('events.bsons').read()):
        db['events'].save(e)
Exemple #3
0
def main(data):
    def str_to_date(s):
        if s is None:
            return None
        return datetime(*strptime(s,'%Y-%m-%d')[:3])
    def year_to_dates(year):
        return (datetime(2003+year,9,1),
            datetime(2004+year,8,31))
    def create_tag(name, humanName, tags=[]):
        return Es.ecol.insert({'types': ['tag'],
                       'names': [name],
                       'humanNames': [{
                           'name': name,
                           'human': humanName}],
                       'tags': tags})
    print 'dropping'
    Es.ecol.drop()
    Es.rcol.drop()
    Es.mcol.drop()
    subscr_Es.ecol.drop()
    subscr_Es.scol.drop()
    print 'creating indices'
    Es.ensure_indices()
    subscr_Es.ensure_indices()
    mod_Es.ensure_indices()
    conv_inst = dict()
    conv_study = dict()
    conv_group = dict()
    conv_group_byname = dict()
    conv_event = dict()
    conv_seat = dict()
    conv_user = dict()
    ignore_groups = frozenset(['leden-oud'])
    ignore_groups_members = frozenset(['leden'])
    ignore_groups_ids = set()
    ignore_groups_members_ids = set()
    year_groups = frozenset(
        ['leden'+str(x) for x in range(1,9)]+
        ['kasco'+str(x) for x in range(1,9)]+
        ['bestuur'+str(x) for x in range(1,9)])
    year_groups_ids = dict()
    year_groups_lut = {}
    print 'initial tags'
    system_tag = create_tag('!system', 'Systeemstempels')
    year_overrides_tag = create_tag('!year-overrides',
            'Jaarlidmaatschapstempels', [system_tag])
    virtual_group_tag = create_tag("!virtual-group",
            'Virtuele groep', [system_tag])
    sofa_brand_tag = create_tag("!sofa-brand",
            'Sofa merk', [system_tag])
    year_group_tag = create_tag("!year-group", 'Jaargroep',
            [system_tag])
    for i in xrange(1,9):
        Es.ecol.insert({'types': ['tag'],
                'humanNames': [{'human': 'Wel jaar %s' % i}],
                'year-override': {'year': i,
                          'type': True},
                'tags': [year_overrides_tag]})
        Es.ecol.insert({'types': ['tag'],
                'humanNames': [{'human': 'Niet jaar %s' % i}],
                'year-override': {'year': i,
                          'type': False},
                'tags': [year_overrides_tag]})
    print 'institutes'
    for m in data['EduInstitute']:
        n = {   'types': ['institute'],
            'humanNames': [{'human': m['name']}]}
        conv_inst[m['id']] = Es.ecol.insert(n)
    print 'studies'
    for m in data['Study']:
        n = {   'types': ['study'],
            'humanNames': [{'human': m['name']}]}
        conv_study[m['id']] = Es.ecol.insert(n)
    print 'initial groups'
    conv_group_byname['bestuur'] = {
        'id': Es.ecol.insert({
            'types': ['group'],
            'names': ['bestuur'],
            'tags': [year_group_tag],
            'humanNames': [{
                'name': 'bestuur',
                'human': 'Bestuur',
                'genitive_prefix': 'van het'}],
            'description': "Het bestuur"}),
        'name': 'bestuur',
        'humanName': 'Bestuur'}
    conv_group_byname['kasco'] = {
        'id': Es.ecol.insert({
            'types': ['group'],
            'names': ['kasco'],
            'tags': [year_group_tag],
            'humanNames': [{
                'name': 'kasco',
                'human': 'Kascontrolecommissie',
                'genitive_prefix': 'van de'}],
            'description': "De kascontrolecommissie"}),
        'name': 'kasco',
        'humanName': 'Bestuur'}
    print 'groups'
    for m in data['OldKnGroup']:
        if m['name'] in ignore_groups:
            ignore_groups_ids.add(m['id'])
            continue
        if m['name'] in ignore_groups_members:
            ignore_groups_members_ids.add(m['id'])
        if m['name'] in year_groups:
            year_groups_ids[m['id']] = m['name']
            group = m['name'][:-1]
            year = int(m['name'][-1:])
            year_groups_lut[m['id']] = (group, year)
            continue
        if m['name'] == 'leden': m['isVirtual'] = False # fix for leden
        n = {   'types': ['tag' if m['isVirtual'] else 'group'],
            'names': [m['name']],
            'humanNames': [{
                'name': m['name'],
                'human': m['humanName'],
                'genitive_prefix': m['genitive_prefix']
                }],
            'description': m['description'],
            'temp':{
                'is_virtual': m['isVirtual']
            }
            }
        conv_group[m['id']] = {'id': Es.ecol.insert(n),
                       'name': m['name'],
                       'humanName': m['humanName']}
        conv_group_byname[m['name']] = conv_group[m['id']]
    print 'group hierarchy'
    for m in data['OldKnGroup']:
        if m['name'] in year_groups or m['name'] in ignore_groups:
            continue
        if m['parent'] is not None:
            if not m['parent'] in conv_group:
                print " %s was orphaned" % m['name']
                continue
            Es.ecol.update({'_id': conv_group[m['id']]['id']},
                {'$push': {'tags': conv_group[
                            m['parent']]['id']}})
    print 'users'
    for m in data['OldKnUser']:
        bits = m['password'].split('$')
        if len(bits) == 3:
            pwd = {'algorithm': bits[0],
                   'salt': bits[1],
                   'hash': bits[2]}
        else:
            pwd = None
        n = {
            'types': ['user'],
            'names': [m['username']],
            'humanNames': [{'human': m['first_name'] + ' ' +
                         m['last_name']}],
            'person': {
                'titles': [],
                'nick': m['first_name'],
                'given': None,
                'family': m['last_name'],
                'gender': m['gender'],
                'dateOfBirth': str_to_date(m['dateOfBirth'])
            },
            'emailAddresses': [
                {'email': m['email'],
                 'from': DT_MIN,
                 'until': DT_MAX
                }],
            'addresses': [
                {'street': m['addr_street'],
                 'number': m['addr_number'],
                 'zip': m['addr_zipCode'],
                 'city': m['addr_city'],
                 'from': DT_MIN,
                 'until': DT_MAX
                }],
            'telephones': [
                {'number': m['telephone'],
                 'from': DT_MIN,
                 'until': DT_MAX}],
            'studies': [
                {'institute': conv_inst.get(m['institute']),
                 'study': conv_study.get(m['study']),
                 'from': DT_MIN,
                 'until': DT_MAX,
                 'number': m['studentNumber']}
            ],
            'temp': {
                'oud': m['in_oud'],
                'aan': m['in_aan'],
                'incasso': m['got_incasso'],
                'joined': m['dateJoined'],
                'remarks': m['remarks']
            },
            'is_active': m['is_active'],
            'password': pwd
            }
        conv_user[m['id']] = Es.ecol.insert(n)
        for g in m['groups']:
            if g in ignore_groups_ids or \
                    g in ignore_groups_members_ids:
                continue
            if g in year_groups_ids:
                gname, year = year_groups_lut[g]
                f, u = year_to_dates(year)
                Es.rcol.insert({
                    'with': conv_group_byname[gname]['id'],
                    'who': conv_user[m['id']],
                    'from': f,
                    'until': u,
                    'how': None})
                continue
            Es.rcol.insert({
                'with': conv_group[g]['id'],
                'who': conv_user[m['id']],
                'from': DT_MIN,
                'until': DT_MAX,
                'how': None})
    print 'brands'
    for m in data['OldSeat']:
        if m['name'] == 'deelhoofd':
            m['name'] = 'graficideelhoofd'
        if m['name'] in conv_seat:
            continue
        n = {'types': ['brand'],
             'names': [m['name']],
             'tags': [sofa_brand_tag],
             'humanNames': [{'name': m['name'],
                     'human': m['humanName']}]}
        conv_seat[m['name']] = {'id': Es.ecol.insert(n)}
    print 'seats'
    for m in data['OldSeat']:
        if m['group'] in year_groups_ids:
            gname = year_groups_ids[m['group']]
            gdat = conv_group_byname[gname[:-1]]
            _from, until = year_to_dates(int(gname[-1:]))
        else:
            gdat = conv_group[m['group']]
            _from, until = DT_MIN, DT_MAX
        n = {'types': ['group'],
             'names': [gdat['name'] + '-' + m['name']],
             'description': [m['description']],
             'tags': [virtual_group_tag],
             'virtual': {
                 'type': 'sofa',
                 'with': gdat['id'],
                 'how': conv_seat[m['name']]['id']},
             'humanNames': [{
                    'name': gdat['name'] + '-' + m['name'],
                'human': m['humanName'] +
                    ' ' + gdat['humanName']}]}
        i = Es.ecol.insert(n)
        Es.rcol.insert({'who': conv_user[m['user']],
                'from': _from,
                'until': until,
                'how': conv_seat[m['name']]['id'],
                'with': gdat['id']})
    print 'merging relations'
    print ' list until'
    lut = dict()
    plan_changes = dict()
    plan_remove = set()
    for r in Es.rcol.find({'until': {'$lt': DT_MAX}}):
        lut[r['until'] + timedelta(1,0), r['with'],
                r['how'], r['who']] = r['_id']
    print ' crossreference from'
    for r in Es.rcol.find({'from': {'$gt': DT_MIN}}):
        n = (r['from'], r['with'], r['how'], r['who'])
        if n not in lut:
            continue
        plan_changes[lut[n]] = (r['until'], r['_id'])
        plan_remove.add(r['_id'])
    print ' transitive closure of plan'
    print 'small final tweaks to groups'
    Es.ecol.update({'names': 'leden'}, {'$push': {'tags': year_group_tag}})
    done = False
    while not done:
        done = True
        for k,v in plan_changes.iteritems():
            if v[1] in plan_changes:
                plan_changes[k] = plan_changes[v[1]]
                del plan_changes[v[1]]
                done = False
                break
    print ' execute'
    for r in plan_remove:
        Es.rcol.remove({'_id': r})
    for k,v in plan_changes.iteritems():
        Es.rcol.update({'_id': k}, {'$set': {'until': v[0]}})
    print 'event'
    for m in data['Event']:
        if m['owner'] not in conv_group:
            gname = year_groups_lut[m['owner']][0]
            gid = conv_group_byname[gname]['id']
        else:
            gid = conv_group[m['owner']]['id']
        conv_event[m['id']] = subscr_Es.ecol.insert({
            'mailBody': m['mailBody'],
            'humanName': m['humanName'],
            'description': m['description'],
            'cost': m['cost'],
            'is_open': m['is_open'],
            'owner': gid,
            'name': m['name']})
    print 'event subscriptions'
    for m in data['EventSubscription']:
        subscr_Es.scol.insert({
            'event': conv_event[m['event']],
            'userNotes': m['userNotes'],
            'debit': m['debit'],
            'user': conv_user[m['user']]})
    print 'giedo updatedb'
    update_db(None)
    print 'alias'
    print ' ids_by_names'
    name2id = Es.ids_by_names()
    print ' to graph'
    alias_graph = {}
    for m in data['Alias']:
        alias_graph[m['source']] = m['target']
    print ' tarjan'
    for scc in tarjan(alias_graph):
        assert len(scc) == 1
        src = scc[0]
        if src in name2id:
            continue
        if not src in alias_graph:
            continue
        if not alias_graph[src] in name2id:
            print '  ? %s -> %s' % (src, alias_graph[src])
            continue
        name2id[src] = name2id[alias_graph[src]]
        Es.ecol.update({'names': alias_graph[src]},
                   {'$push': {'names': src}})
def main(data):
    def str_to_date(s):
        if s is None:
            return None
        return datetime(*strptime(s, "%Y-%m-%d")[:3])

    def year_to_dates(year):
        return (datetime(2003 + year, 9, 1), datetime(2004 + year, 8, 31))

    def create_tag(name, humanName, tags=[]):
        return Es.ecol.insert(
            {"types": ["tag"], "names": [name], "humanNames": [{"name": name, "human": humanName}], "tags": tags}
        )

    print "dropping"
    Es.ecol.drop()
    Es.rcol.drop()
    Es.mcol.drop()
    subscr_Es.ecol.drop()
    subscr_Es.scol.drop()
    print "creating indices"
    Es.ensure_indices()
    subscr_Es.ensure_indices()
    mod_Es.ensure_indices()
    conv_inst = dict()
    conv_study = dict()
    conv_group = dict()
    conv_group_byname = dict()
    conv_event = dict()
    conv_seat = dict()
    conv_user = dict()
    ignore_groups = frozenset(["leden-oud"])
    ignore_groups_members = frozenset(["leden"])
    ignore_groups_ids = set()
    ignore_groups_members_ids = set()
    year_groups = frozenset(
        ["leden" + str(x) for x in range(1, 9)]
        + ["kasco" + str(x) for x in range(1, 9)]
        + ["bestuur" + str(x) for x in range(1, 9)]
    )
    year_groups_ids = dict()
    year_groups_lut = {}
    print "initial tags"
    system_tag = create_tag("!system", "Systeemstempels")
    year_overrides_tag = create_tag("!year-overrides", "Jaarlidmaatschapstempels", [system_tag])
    virtual_group_tag = create_tag("!virtual-group", "Virtuele groep", [system_tag])
    sofa_brand_tag = create_tag("!sofa-brand", "Sofa merk", [system_tag])
    year_group_tag = create_tag("!year-group", "Jaargroep", [system_tag])
    for i in xrange(1, 9):
        Es.ecol.insert(
            {
                "types": ["tag"],
                "humanNames": [{"human": "Wel jaar %s" % i}],
                "year-override": {"year": i, "type": True},
                "tags": [year_overrides_tag],
            }
        )
        Es.ecol.insert(
            {
                "types": ["tag"],
                "humanNames": [{"human": "Niet jaar %s" % i}],
                "year-override": {"year": i, "type": False},
                "tags": [year_overrides_tag],
            }
        )
    print "institutes"
    for m in data["EduInstitute"]:
        n = {"types": ["institute"], "humanNames": [{"human": m["name"]}]}
        conv_inst[m["id"]] = Es.ecol.insert(n)
    print "studies"
    for m in data["Study"]:
        n = {"types": ["study"], "humanNames": [{"human": m["name"]}]}
        conv_study[m["id"]] = Es.ecol.insert(n)
    print "initial groups"
    conv_group_byname["bestuur"] = {
        "id": Es.ecol.insert(
            {
                "types": ["group"],
                "names": ["bestuur"],
                "tags": [year_group_tag],
                "humanNames": [{"name": "bestuur", "human": "Bestuur", "genitive_prefix": "van het"}],
                "description": "Het bestuur",
            }
        ),
        "name": "bestuur",
        "humanName": "Bestuur",
    }
    conv_group_byname["kasco"] = {
        "id": Es.ecol.insert(
            {
                "types": ["group"],
                "names": ["kasco"],
                "tags": [year_group_tag],
                "humanNames": [{"name": "kasco", "human": "Kascontrolecommissie", "genitive_prefix": "van de"}],
                "description": "De kascontrolecommissie",
            }
        ),
        "name": "kasco",
        "humanName": "Bestuur",
    }
    print "groups"
    for m in data["OldKnGroup"]:
        if m["name"] in ignore_groups:
            ignore_groups_ids.add(m["id"])
            continue
        if m["name"] in ignore_groups_members:
            ignore_groups_members_ids.add(m["id"])
        if m["name"] in year_groups:
            year_groups_ids[m["id"]] = m["name"]
            group = m["name"][:-1]
            year = int(m["name"][-1:])
            year_groups_lut[m["id"]] = (group, year)
            continue
        if m["name"] == "leden":
            m["isVirtual"] = False  # fix for leden
        n = {
            "types": ["tag" if m["isVirtual"] else "group"],
            "names": [m["name"]],
            "humanNames": [{"name": m["name"], "human": m["humanName"], "genitive_prefix": m["genitive_prefix"]}],
            "description": m["description"],
            "temp": {"is_virtual": m["isVirtual"]},
        }
        conv_group[m["id"]] = {"id": Es.ecol.insert(n), "name": m["name"], "humanName": m["humanName"]}
        conv_group_byname[m["name"]] = conv_group[m["id"]]
    print "group hierarchy"
    for m in data["OldKnGroup"]:
        if m["name"] in year_groups or m["name"] in ignore_groups:
            continue
        if m["parent"] is not None:
            if not m["parent"] in conv_group:
                print " %s was orphaned" % m["name"]
                continue
            Es.ecol.update({"_id": conv_group[m["id"]]["id"]}, {"$push": {"tags": conv_group[m["parent"]]["id"]}})
    print "users"
    for m in data["OldKnUser"]:
        bits = m["password"].split("$")
        if len(bits) == 3:
            pwd = {"algorithm": bits[0], "salt": bits[1], "hash": bits[2]}
        else:
            pwd = None
        n = {
            "types": ["user"],
            "names": [m["username"]],
            "humanNames": [{"human": m["first_name"] + " " + m["last_name"]}],
            "person": {
                "titles": [],
                "nick": m["first_name"],
                "given": None,
                "family": m["last_name"],
                "gender": m["gender"],
                "dateOfBirth": str_to_date(m["dateOfBirth"]),
            },
            "emailAddresses": [{"email": m["email"], "from": DT_MIN, "until": DT_MAX}],
            "addresses": [
                {
                    "street": m["addr_street"],
                    "number": m["addr_number"],
                    "zip": m["addr_zipCode"],
                    "city": m["addr_city"],
                    "from": DT_MIN,
                    "until": DT_MAX,
                }
            ],
            "telephones": [{"number": m["telephone"], "from": DT_MIN, "until": DT_MAX}],
            "studies": [
                {
                    "institute": conv_inst.get(m["institute"]),
                    "study": conv_study.get(m["study"]),
                    "from": DT_MIN,
                    "until": DT_MAX,
                    "number": m["studentNumber"],
                }
            ],
            "temp": {
                "oud": m["in_oud"],
                "aan": m["in_aan"],
                "incasso": m["got_incasso"],
                "joined": m["dateJoined"],
                "remarks": m["remarks"],
            },
            "is_active": m["is_active"],
            "password": pwd,
        }
        conv_user[m["id"]] = Es.ecol.insert(n)
        for g in m["groups"]:
            if g in ignore_groups_ids or g in ignore_groups_members_ids:
                continue
            if g in year_groups_ids:
                gname, year = year_groups_lut[g]
                f, u = year_to_dates(year)
                Es.rcol.insert(
                    {
                        "with": conv_group_byname[gname]["id"],
                        "who": conv_user[m["id"]],
                        "from": f,
                        "until": u,
                        "how": None,
                    }
                )
                continue
            Es.rcol.insert(
                {"with": conv_group[g]["id"], "who": conv_user[m["id"]], "from": DT_MIN, "until": DT_MAX, "how": None}
            )
    print "brands"
    for m in data["OldSeat"]:
        if m["name"] == "deelhoofd":
            m["name"] = "graficideelhoofd"
        if m["name"] in conv_seat:
            continue
        n = {
            "types": ["brand"],
            "names": [m["name"]],
            "tags": [sofa_brand_tag],
            "humanNames": [{"name": m["name"], "human": m["humanName"]}],
        }
        conv_seat[m["name"]] = {"id": Es.ecol.insert(n)}
    print "seats"
    for m in data["OldSeat"]:
        if m["group"] in year_groups_ids:
            gname = year_groups_ids[m["group"]]
            gdat = conv_group_byname[gname[:-1]]
            _from, until = year_to_dates(int(gname[-1:]))
        else:
            gdat = conv_group[m["group"]]
            _from, until = DT_MIN, DT_MAX
        n = {
            "types": ["group"],
            "names": [gdat["name"] + "-" + m["name"]],
            "description": [m["description"]],
            "tags": [virtual_group_tag],
            "virtual": {"type": "sofa", "with": gdat["id"], "how": conv_seat[m["name"]]["id"]},
            "humanNames": [{"name": gdat["name"] + "-" + m["name"], "human": m["humanName"] + " " + gdat["humanName"]}],
        }
        i = Es.ecol.insert(n)
        Es.rcol.insert(
            {
                "who": conv_user[m["user"]],
                "from": _from,
                "until": until,
                "how": conv_seat[m["name"]]["id"],
                "with": gdat["id"],
            }
        )
    print "merging relations"
    print " list until"
    lut = dict()
    plan_changes = dict()
    plan_remove = set()
    for r in Es.rcol.find({"until": {"$lt": DT_MAX}}):
        lut[r["until"] + timedelta(1, 0), r["with"], r["how"], r["who"]] = r["_id"]
    print " crossreference from"
    for r in Es.rcol.find({"from": {"$gt": DT_MIN}}):
        n = (r["from"], r["with"], r["how"], r["who"])
        if n not in lut:
            continue
        plan_changes[lut[n]] = (r["until"], r["_id"])
        plan_remove.add(r["_id"])
    print " transitive closure of plan"
    print "small final tweaks to groups"
    Es.ecol.update({"names": "leden"}, {"$push": {"tags": year_group_tag}})
    done = False
    while not done:
        done = True
        for k, v in plan_changes.iteritems():
            if v[1] in plan_changes:
                plan_changes[k] = plan_changes[v[1]]
                del plan_changes[v[1]]
                done = False
                break
    print " execute"
    for r in plan_remove:
        Es.rcol.remove({"_id": r})
    for k, v in plan_changes.iteritems():
        Es.rcol.update({"_id": k}, {"$set": {"until": v[0]}})
    print "event"
    for m in data["Event"]:
        if m["owner"] not in conv_group:
            gname = year_groups_lut[m["owner"]][0]
            gid = conv_group_byname[gname]["id"]
        else:
            gid = conv_group[m["owner"]]["id"]
        conv_event[m["id"]] = subscr_Es.ecol.insert(
            {
                "mailBody": m["mailBody"],
                "humanName": m["humanName"],
                "description": m["description"],
                "cost": m["cost"],
                "is_open": m["is_open"],
                "owner": gid,
                "name": m["name"],
            }
        )
    print "event subscriptions"
    for m in data["EventSubscription"]:
        subscr_Es.scol.insert(
            {
                "event": conv_event[m["event"]],
                "userNotes": m["userNotes"],
                "debit": m["debit"],
                "user": conv_user[m["user"]],
            }
        )
    print "giedo updatedb"
    update_db(None)
    print "alias"
    print " ids_by_names"
    name2id = Es.ids_by_names()
    print " to graph"
    alias_graph = {}
    for m in data["Alias"]:
        alias_graph[m["source"]] = m["target"]
    print " tarjan"
    for scc in tarjan(alias_graph):
        assert len(scc) == 1
        src = scc[0]
        if src in name2id:
            continue
        if not src in alias_graph:
            continue
        if not alias_graph[src] in name2id:
            print "  ? %s -> %s" % (src, alias_graph[src])
            continue
        name2id[src] = name2id[alias_graph[src]]
        Es.ecol.update({"names": alias_graph[src]}, {"$push": {"names": src}})