def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, 'w', cachesize=1024*1024*1024) from infogami.infobase.utils import flatten_dict indexable_keys = set([ "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times" # work ]) for type, key, revision, timestamp, json in read_tsv(dump_file): db[key] = json d = simplejson.loads(json) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = '/' + v.lower().replace(" ", "_") dbkey = web.safestr('by_%s%s' % (k, v)) if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")
def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, 'w', cachesize=1024 * 1024 * 1024) from infogami.infobase.utils import flatten_dict indexable_keys = set([ "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times" # work ]) for type, key, revision, timestamp, json in read_tsv(dump_file): db[key] = json d = simplejson.loads(json) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = '/' + v.lower().replace(" ", "_") dbkey = web.safestr('by_%s%s' % (k, v)) if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")
def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, "w", cachesize=1024 * 1024 * 1024) indexable_keys = { "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times", # work } for type, key, revision, timestamp, json_data in read_tsv(dump_file): db[key] = json_data d = json.loads(json_data) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = "/" + v.lower().replace(" ", "_") dbkey = web.safestr(f"by_{k}{v}") if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")