def process_redirect_dump(writer, redirects_dump): import bsddb db = bsddb.btopen('solrdump/redirects.db', 'w', cachesize=1024 * 1024 * 1024) for type, key, revision, timestamp, json_data in read_tsv(redirects_dump): d = json.loads(json_data) if not key.startswith(("/authors/", "/works/")): continue location = d.get('location') if location: # Old redirects still start with /a/ instead of /authors/. location = location.replace("/a/", "/authors/") db[key] = location for key in db: if key.startswith("/works/"): redirect = find_redirect(db, key) if redirect: writer.write([(redirect, "redirect", key)]) return db
def process_triples(path): """Takes a file with triples, sort it using first column and groups it by first column.""" print("processing triples from", path) cmd = ["sort", path] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) for key, chunk in itertools.groupby(read_tsv(p.stdout), lambda t: t[0]): d = collections.defaultdict(list) for k, name, value in chunk: if name in ['json', 'edition', 'author']: value = json.loads(value) d[name].append(value) yield key, d
def process_author_dump(writer, authors_dump): import bsddb db = bsddb.btopen('solrdump/authors.db', 'w', cachesize=1024 * 1024 * 1024) properties = ['key', 'name', 'alternate_names', 'personal_name'] for type, key, revision, timestamp, json_data in read_tsv(authors_dump): author = json.loads(json_data) olid = key.split("/")[-1] db[olid] = json.dumps(subdict(author, properties)) writer.write(process_author(author)) return db
def main(filename): for tokens in read_tsv(filename): json = tokens[-1] doc = simplejson.loads(json) cover = doc.get('covers') and doc.get('covers')[0] isbns = doc.get('isbn_10', []) + doc.get('isbn_13', []) key = doc['key'] key = web.safestr(key) isbns = (web.safestr(isbn) for isbn in isbns) try: if cover and cover > 0: print "\t".join([str(cover), key, ",".join(isbns)]) except: print >> sys.stderr, doc print >> sys.stderr, (key, cover, isbns) raise
def main(filename): for tokens in read_tsv(filename): json = tokens[-1] doc = simplejson.loads(json) cover = doc.get('covers') and doc.get('covers')[0] isbns = doc.get('isbn_10', []) + doc.get('isbn_13', []) key = doc['key'] key = web.safestr(key) isbns = (web.safestr(isbn) for isbn in isbns) try: if cover and cover > 0: print "\t".join([str(cover), key, ",".join(isbns)]) except: print >> sys.stderr, doc print >> sys.stderr, (key, cover, isbns) raise
def process_edition_dump(writer, editions_dump): for type, key, revision, timestamp, json_data in read_tsv(editions_dump): doc = json.loads(json_data) writer.write(process_edition(doc))
def process_work_dump(writer, works_dump, author_db, redirect_db): for type, key, revision, timestamp, json_data in read_tsv(works_dump): doc = json.loads(json_data) writer.write(process_work(doc, author_db, redirect_db))