def write_edition(loc, edition): add_lang(edition) q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print a raise print 'ret:', ret assert isinstance(ret, basestring) # assert ret['status'] == 'ok' # assert 'created' in ret and len(ret['created']) == 1 authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors for attempt in range(5): if attempt > 0: print 'retrying' try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(10) continue except: # httplib.BadStatusLine print q raise break print 'ret:', ret assert isinstance(ret, basestring) # assert ret['status'] == 'ok' # assert 'created' in ret # editions = [i for i in ret['created'] if i.startswith('/b/OL')] # assert len(editions) == 1 key = ret # get key from return pool.update(key, q)
def write_edition(ia, edition): loc = 'ia:' + ia add_lang(edition) q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print a raise print 'ret:', ret assert isinstance(ret, basestring) authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors for attempt in range(50): if attempt > 0: print 'retrying' try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print q raise break print 'ret:', ret assert isinstance(ret, basestring) key = ret pool.update(key, q) print 'add_cover_image' add_cover_image(key, ia)
def write_edition(loc, edition): add_lang(edition) q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print(a) raise print('ret:', ret) assert isinstance(ret, basestring) authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors for attempt in range(50): if attempt > 0: print('retrying') try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print(q) raise break print('ret:', ret) assert isinstance(ret, basestring) key = ret pool.update(key, q)
def write_edition(loc, edition): add_lang(edition) q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print(a) raise print('ret:', ret) assert isinstance(ret, six.string_types) authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors for attempt in range(50): if attempt > 0: print('retrying') try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print(q) raise break print('ret:', ret) assert isinstance(ret, six.string_types) key = ret pool.update(key, q)
def write_edition(ia, edition, rec): loc = 'ia:' + ia if ia == 'munkai00apor': edition['languages'] = [ { 'key': '/languages/lat' }, { 'key': '/languages/hun' }, ] elif ia == 'fzfasp00helt': edition['languages'] = [{'key': '/languages/hun'}] elif ia == 'coursetconfren00sema': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'repertoiredepein02rein': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'repertoiredepein01rein': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'cihm_39338': edition['languages'] = [{'key': '/languages/ger'}] elif ia == 'ofilhoprdigodr00mano': edition['languages'] = [{'key': '/languages/por'}] elif ia == 'nekaroronneteyer00hill': edition['languages'] = [{'key': '/languages/moh'}] elif ia == 'adventuresofamer00kouw': edition['languages'] = [{'key': '/languages/eng'}] elif ia == 'goldentreasury00wrig': edition['languages'] = [{'key': '/languages/grc'}] elif ia == 'dasrmischepriv00rein': edition['languages'] = [{'key': '/languages/ger'}] elif ia == 'lespritdelaligu02anqu': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'derelephantenord00berl': del edition['languages'] q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print(a) raise print('ret:', ret) assert isinstance(ret, basestring) authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors wkey = None subjects = subjects_for_work(rec) subjects.setdefault('subjects', []).append('Accessible book') if 'printdisabled' in collections: subjects['subjects'].append('Protected DAISY') elif 'lendinglibrary' in collections: subjects['subjects'] += ['Protected DAISY', 'Lending library'] elif 'inlibrary' in collections: subjects['subjects'] += ['Protected DAISY', 'In library'] if 'authors' in q: wkey = find_matching_work(q) if wkey: w = ol.get(wkey) need_update = False for k, subject_list in subjects.items(): for s in subject_list: if s not in w.get(k, []): w.setdefault(k, []).append(s) need_update = True if need_update: ol.save(wkey, w, 'add subjects from new record') else: w = { 'type': '/type/work', 'title': q['title'], } if 'authors' in q: w['authors'] = [{ 'type': '/type/author_role', 'author': akey } for akey in q['authors']] w.update(subjects) wkey = ol.new(w, comment='initial import') q['works'] = [{'key': wkey}] for attempt in range(50): if attempt > 0: print('retrying') try: pprint(q) ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print(q) raise break print('ret:', ret) assert isinstance(ret, basestring) key = '/b/' + re_edition_key.match(ret).group(1) pool.update(key, q) print('add_cover_image') t = threading.Thread(target=add_cover_image, args=(ret, ia)) t.start() return print('run work finder') # too slow for a in authors: akey = a['key'] title_redirects = find_title_redirects(akey) works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects) works = list(works) updated = update_works(akey, works, do_updates=True)
def write_edition(ia, edition, rec): loc = 'ia:' + ia if ia == 'munkai00apor': edition['languages'] = [ {'key': '/languages/lat'}, {'key': '/languages/hun'}, ] elif ia == 'fzfasp00helt': edition['languages'] = [{'key': '/languages/hun'}] elif ia == 'coursetconfren00sema': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'repertoiredepein02rein': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'repertoiredepein01rein': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'cihm_39338': edition['languages'] = [{'key': '/languages/ger'}] elif ia == 'ofilhoprdigodr00mano': edition['languages'] = [{'key': '/languages/por'}] elif ia == 'nekaroronneteyer00hill': edition['languages'] = [{'key': '/languages/moh'}] elif ia == 'adventuresofamer00kouw': edition['languages'] = [{'key': '/languages/eng'}] elif ia == 'goldentreasury00wrig': edition['languages'] = [{'key': '/languages/grc'}] elif ia == 'dasrmischepriv00rein': edition['languages'] = [{'key': '/languages/ger'}] elif ia == 'lespritdelaligu02anqu': edition['languages'] = [{'key': '/languages/fre'}] elif ia == 'derelephantenord00berl': del edition['languages'] q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print a raise print 'ret:', ret assert isinstance(ret, basestring) authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors wkey = None subjects = subjects_for_work(rec) subjects.setdefault('subjects', []).append('Accessible book') if 'printdisabled' in collections: subjects['subjects'].append('Protected DAISY') elif 'lendinglibrary' in collections: subjects['subjects'] += ['Protected DAISY', 'Lending library'] elif 'inlibrary' in collections: subjects['subjects'] += ['Protected DAISY', 'In library'] if 'authors' in q: wkey = find_matching_work(q) if wkey: w = ol.get(wkey) need_update = False for k, subject_list in subjects.items(): for s in subject_list: if s not in w.get(k, []): w.setdefault(k, []).append(s) need_update = True if need_update: ol.save(wkey, w, 'add subjects from new record') else: w = { 'type': '/type/work', 'title': q['title'], } if 'authors' in q: w['authors'] = [{'type':'/type/author_role', 'author': akey} for akey in q['authors']] w.update(subjects) wkey = ol.new(w, comment='initial import') q['works'] = [{'key': wkey}] for attempt in range(50): if attempt > 0: print 'retrying' try: pprint(q) ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print q raise break print 'ret:', ret assert isinstance(ret, basestring) key = '/b/' + re_edition_key.match(ret).group(1) pool.update(key, q) print 'add_cover_image' t = threading.Thread(target=add_cover_image, args=(ret, ia)) t.start() return print 'run work finder' # too slow for a in authors: akey = a['key'] title_redirects = find_title_redirects(akey) works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects) works = list(works) updated = update_works(akey, works, do_updates=True)
def write_edition(ia, edition, rec): loc = "ia:" + ia if ia == "munkai00apor": edition["languages"] = [{"key": "/languages/lat"}, {"key": "/languages/hun"}] elif ia == "coursetconfren00sema": edition["languages"] = [{"key": "/languages/fre"}] elif ia == "repertoiredepein02rein": edition["languages"] = [{"key": "/languages/fre"}] elif ia == "repertoiredepein01rein": edition["languages"] = [{"key": "/languages/fre"}] elif ia == "cihm_39338": edition["languages"] = [{"key": "/languages/ger"}] elif ia == "ofilhoprdigodr00mano": edition["languages"] = [{"key": "/languages/por"}] elif ia == "nekaroronneteyer00hill": edition["languages"] = [{"key": "/languages/moh"}] elif ia == "adventuresofamer00kouw": edition["languages"] = [{"key": "/languages/eng"}] elif ia == "goldentreasury00wrig": edition["languages"] = [{"key": "/languages/grc"}] elif ia == "dasrmischepriv00rein": edition["languages"] = [{"key": "/languages/ger"}] elif ia == "derelephantenord00berl": del edition["languages"] q = build_query(loc, edition) authors = [] for a in q.get("authors", []): if "key" in a: authors.append({"key": a["key"]}) else: try: ret = ol.new(a, comment="new author") except: print a raise print "ret:", ret assert isinstance(ret, basestring) authors.append({"key": ret}) q["source_records"] = [loc] if authors: q["authors"] = authors wkey = None subjects = subjects_for_work(rec) subjects.setdefault("subjects", []).append("Accessible book") if "authors" in q: wkey = find_matching_work(q) if wkey: w = ol.get(wkey) need_update = False for k, subject_list in subjects.items(): for s in subject_list: if s not in w.get(k, []): w.setdefault(k, []).append(s) need_update = True if need_update: ol.save(wkey, w, "add subjects from new record") else: w = {"type": "/type/work", "title": q["title"]} if "authors" in q: w["authors"] = [{"type": "/type/author_role", "author": akey} for akey in q["authors"]] w.update(subjects) wkey = ol.new(w, comment="initial import") q["works"] = [{"key": wkey}] for attempt in range(50): if attempt > 0: print "retrying" try: pprint(q) ret = ol.new(q, comment="initial import") except httplib.BadStatusLine: sleep(30) continue except: # httplib.BadStatusLine print q raise break print "ret:", ret assert isinstance(ret, basestring) key = "/b/" + re_edition_key.match(ret).group(1) pool.update(key, q) print "add_cover_image" t = threading.Thread(target=add_cover_image, args=(ret, ia)) t.start() return print "run work finder" # too slow for a in authors: akey = a["key"] title_redirects = find_title_redirects(akey) works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects) works = list(works) updated = update_works(akey, works, do_updates=True)
def write_edition(loc, edition): q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print(a) raise print('ret:', ret) assert isinstance(ret, basestring) # assert ret['status'] == 'ok' # assert 'created' in ret and len(ret['created']) == 1 authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors wkey = None subjects = subjects_for_work(rec) if 'authors' in q: wkey = find_matching_work(q) if wkey: w = ol.get(wkey) need_update = False for k, subject_list in subjects.items(): for s in subject_list: if s not in w.get(k, []): w.setdefault(k, []).append(s) need_update = True if need_update: ol.save(wkey, w, 'add subjects from new record') else: w = { 'type': '/type/work', 'title': q['title'], } if 'authors' in q: w['authors'] = [{ 'type': '/type/author_role', 'author': akey } for akey in q['authors']] w.update(subjects) wkey = ol.new(w, comment='initial import') q['works'] = [{'key': wkey}] for attempt in range(5): if attempt > 0: print('retrying') try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(10) continue except: # httplib.BadStatusLine print(q) raise break print('ret:', ret) assert isinstance(ret, basestring) key = '/b/' + re_edition_key.match(ret).group(1) # assert ret['status'] == 'ok' # assert 'created' in ret # editions = [i for i in ret['created'] if i.startswith('/b/OL')] # assert len(editions) == 1 # get key from return pool.update(key, q) return for a in authors: akey = a['key'] title_redirects = find_title_redirects(akey) works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects) works = list(works) updated = update_works(akey, works, do_updates=True)
def write_edition(loc, edition): q = build_query(loc, edition) authors = [] for a in q.get('authors', []): if 'key' in a: authors.append({'key': a['key']}) else: try: ret = ol.new(a, comment='new author') except: print(a) raise print('ret:', ret) assert isinstance(ret, six.string_types) # assert ret['status'] == 'ok' # assert 'created' in ret and len(ret['created']) == 1 authors.append({'key': ret}) q['source_records'] = [loc] if authors: q['authors'] = authors wkey = None subjects = subjects_for_work(rec) if 'authors' in q: wkey = find_matching_work(q) if wkey: w = ol.get(wkey) need_update = False for k, subject_list in subjects.items(): for s in subject_list: if s not in w.get(k, []): w.setdefault(k, []).append(s) need_update = True if need_update: ol.save(wkey, w, 'add subjects from new record') else: w = { 'type': '/type/work', 'title': q['title'], } if 'authors' in q: w['authors'] = [{'type':'/type/author_role', 'author': akey} for akey in q['authors']] w.update(subjects) wkey = ol.new(w, comment='initial import') q['works'] = [{'key': wkey}] for attempt in range(5): if attempt > 0: print('retrying') try: ret = ol.new(q, comment='initial import') except httplib.BadStatusLine: sleep(10) continue except: # httplib.BadStatusLine print(q) raise break print('ret:', ret) assert isinstance(ret, six.string_types) key = '/b/' + re_edition_key.match(ret).group(1) # assert ret['status'] == 'ok' # assert 'created' in ret # editions = [i for i in ret['created'] if i.startswith('/b/OL')] # assert len(editions) == 1 # get key from return pool.update(key, q) return for a in authors: akey = a['key'] title_redirects = find_title_redirects(akey) works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects) works = list(works) updated = update_works(akey, works, do_updates=True)