Ejemplo n.º 1
0
def write_edition(loc, edition):
    add_lang(edition)
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print a
                raise
            print 'ret:', ret
            assert isinstance(ret, basestring)
#            assert ret['status'] == 'ok'
#            assert 'created' in ret and len(ret['created']) == 1
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    for attempt in range(5):
        if attempt > 0:
            print 'retrying'
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(10)
            continue
        except: # httplib.BadStatusLine
            print q
            raise
        break
    print 'ret:', ret
    assert isinstance(ret, basestring)
#    assert ret['status'] == 'ok'
#    assert 'created' in ret
#    editions = [i for i in ret['created'] if i.startswith('/b/OL')]
#    assert len(editions) == 1
    key = ret
    # get key from return
    pool.update(key, q)
Ejemplo n.º 2
0
def write_edition(ia, edition):
    loc = 'ia:' + ia
    add_lang(edition)
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print a
                raise
            print 'ret:', ret
            assert isinstance(ret, basestring)
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    for attempt in range(50):
        if attempt > 0:
            print 'retrying'
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except: # httplib.BadStatusLine
            print q
            raise
        break
    print 'ret:', ret
    assert isinstance(ret, basestring)
    key = ret
    pool.update(key, q)

    print 'add_cover_image'
    add_cover_image(key, ia)
Ejemplo n.º 3
0
def write_edition(loc, edition):
    add_lang(edition)
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print(a)
                raise
            print('ret:', ret)
            assert isinstance(ret, basestring)
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    for attempt in range(50):
        if attempt > 0:
            print('retrying')
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except:  # httplib.BadStatusLine
            print(q)
            raise
        break
    print('ret:', ret)
    assert isinstance(ret, basestring)
    key = ret
    pool.update(key, q)
Ejemplo n.º 4
0
def write_edition(loc, edition):
    add_lang(edition)
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print(a)
                raise
            print('ret:', ret)
            assert isinstance(ret, six.string_types)
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    for attempt in range(50):
        if attempt > 0:
            print('retrying')
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except: # httplib.BadStatusLine
            print(q)
            raise
        break
    print('ret:', ret)
    assert isinstance(ret, six.string_types)
    key = ret
    pool.update(key, q)
Ejemplo n.º 5
0
def write_edition(ia, edition, rec):
    loc = 'ia:' + ia
    if ia == 'munkai00apor':
        edition['languages'] = [
            {
                'key': '/languages/lat'
            },
            {
                'key': '/languages/hun'
            },
        ]
    elif ia == 'fzfasp00helt':
        edition['languages'] = [{'key': '/languages/hun'}]
    elif ia == 'coursetconfren00sema':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'repertoiredepein02rein':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'repertoiredepein01rein':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'cihm_39338':
        edition['languages'] = [{'key': '/languages/ger'}]
    elif ia == 'ofilhoprdigodr00mano':
        edition['languages'] = [{'key': '/languages/por'}]
    elif ia == 'nekaroronneteyer00hill':
        edition['languages'] = [{'key': '/languages/moh'}]
    elif ia == 'adventuresofamer00kouw':
        edition['languages'] = [{'key': '/languages/eng'}]
    elif ia == 'goldentreasury00wrig':
        edition['languages'] = [{'key': '/languages/grc'}]
    elif ia == 'dasrmischepriv00rein':
        edition['languages'] = [{'key': '/languages/ger'}]
    elif ia == 'lespritdelaligu02anqu':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'derelephantenord00berl':
        del edition['languages']
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print(a)
                raise
            print('ret:', ret)
            assert isinstance(ret, basestring)
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    wkey = None
    subjects = subjects_for_work(rec)
    subjects.setdefault('subjects', []).append('Accessible book')

    if 'printdisabled' in collections:
        subjects['subjects'].append('Protected DAISY')
    elif 'lendinglibrary' in collections:
        subjects['subjects'] += ['Protected DAISY', 'Lending library']
    elif 'inlibrary' in collections:
        subjects['subjects'] += ['Protected DAISY', 'In library']

    if 'authors' in q:
        wkey = find_matching_work(q)
    if wkey:
        w = ol.get(wkey)
        need_update = False
        for k, subject_list in subjects.items():
            for s in subject_list:
                if s not in w.get(k, []):
                    w.setdefault(k, []).append(s)
                    need_update = True
        if need_update:
            ol.save(wkey, w, 'add subjects from new record')
    else:
        w = {
            'type': '/type/work',
            'title': q['title'],
        }
        if 'authors' in q:
            w['authors'] = [{
                'type': '/type/author_role',
                'author': akey
            } for akey in q['authors']]
        w.update(subjects)

        wkey = ol.new(w, comment='initial import')

    q['works'] = [{'key': wkey}]
    for attempt in range(50):
        if attempt > 0:
            print('retrying')
        try:
            pprint(q)
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except:  # httplib.BadStatusLine
            print(q)
            raise
        break
    print('ret:', ret)
    assert isinstance(ret, basestring)
    key = '/b/' + re_edition_key.match(ret).group(1)
    pool.update(key, q)

    print('add_cover_image')
    t = threading.Thread(target=add_cover_image, args=(ret, ia))
    t.start()
    return

    print('run work finder')

    # too slow
    for a in authors:
        akey = a['key']
        title_redirects = find_title_redirects(akey)
        works = find_works(akey,
                           get_books(akey, books_query(akey)),
                           existing=title_redirects)
        works = list(works)
        updated = update_works(akey, works, do_updates=True)
Ejemplo n.º 6
0
def write_edition(ia, edition, rec):
    loc = 'ia:' + ia
    if ia == 'munkai00apor':
        edition['languages'] = [
            {'key': '/languages/lat'},
            {'key': '/languages/hun'},
        ]
    elif ia == 'fzfasp00helt':
        edition['languages'] = [{'key': '/languages/hun'}]
    elif ia == 'coursetconfren00sema':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'repertoiredepein02rein':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'repertoiredepein01rein':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'cihm_39338':
        edition['languages'] = [{'key': '/languages/ger'}]
    elif ia == 'ofilhoprdigodr00mano':
        edition['languages'] = [{'key': '/languages/por'}]
    elif ia == 'nekaroronneteyer00hill':
        edition['languages'] = [{'key': '/languages/moh'}]
    elif ia == 'adventuresofamer00kouw':
        edition['languages'] = [{'key': '/languages/eng'}]
    elif ia == 'goldentreasury00wrig':
        edition['languages'] = [{'key': '/languages/grc'}]
    elif ia == 'dasrmischepriv00rein':
        edition['languages'] = [{'key': '/languages/ger'}]
    elif ia == 'lespritdelaligu02anqu':
        edition['languages'] = [{'key': '/languages/fre'}]
    elif ia == 'derelephantenord00berl':
        del edition['languages']
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print a
                raise
            print 'ret:', ret
            assert isinstance(ret, basestring)
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    wkey = None
    subjects = subjects_for_work(rec)
    subjects.setdefault('subjects', []).append('Accessible book')

    if 'printdisabled' in collections:
        subjects['subjects'].append('Protected DAISY')
    elif 'lendinglibrary' in collections:
        subjects['subjects'] += ['Protected DAISY', 'Lending library']
    elif 'inlibrary' in collections:
        subjects['subjects'] += ['Protected DAISY', 'In library']

    if 'authors' in q:
        wkey = find_matching_work(q)
    if wkey:
        w = ol.get(wkey)
        need_update = False
        for k, subject_list in subjects.items():
            for s in subject_list:
                if s not in w.get(k, []):
                    w.setdefault(k, []).append(s)
                    need_update = True
        if need_update:
            ol.save(wkey, w, 'add subjects from new record')
    else:
        w = {
            'type': '/type/work',
            'title': q['title'],
        }
        if 'authors' in q:
            w['authors'] = [{'type':'/type/author_role', 'author': akey} for akey in q['authors']]
        w.update(subjects)

        wkey = ol.new(w, comment='initial import')

    q['works'] = [{'key': wkey}]
    for attempt in range(50):
        if attempt > 0:
            print 'retrying'
        try:
            pprint(q)
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except: # httplib.BadStatusLine
            print q
            raise
        break
    print 'ret:', ret
    assert isinstance(ret, basestring)
    key = '/b/' + re_edition_key.match(ret).group(1)
    pool.update(key, q)

    print 'add_cover_image'
    t = threading.Thread(target=add_cover_image, args=(ret, ia))
    t.start()
    return

    print 'run work finder'


    # too slow
    for a in authors:
        akey = a['key']
        title_redirects = find_title_redirects(akey)
        works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects)
        works = list(works)
        updated = update_works(akey, works, do_updates=True)
Ejemplo n.º 7
0
def write_edition(ia, edition, rec):
    loc = "ia:" + ia
    if ia == "munkai00apor":
        edition["languages"] = [{"key": "/languages/lat"}, {"key": "/languages/hun"}]
    elif ia == "coursetconfren00sema":
        edition["languages"] = [{"key": "/languages/fre"}]
    elif ia == "repertoiredepein02rein":
        edition["languages"] = [{"key": "/languages/fre"}]
    elif ia == "repertoiredepein01rein":
        edition["languages"] = [{"key": "/languages/fre"}]
    elif ia == "cihm_39338":
        edition["languages"] = [{"key": "/languages/ger"}]
    elif ia == "ofilhoprdigodr00mano":
        edition["languages"] = [{"key": "/languages/por"}]
    elif ia == "nekaroronneteyer00hill":
        edition["languages"] = [{"key": "/languages/moh"}]
    elif ia == "adventuresofamer00kouw":
        edition["languages"] = [{"key": "/languages/eng"}]
    elif ia == "goldentreasury00wrig":
        edition["languages"] = [{"key": "/languages/grc"}]
    elif ia == "dasrmischepriv00rein":
        edition["languages"] = [{"key": "/languages/ger"}]
    elif ia == "derelephantenord00berl":
        del edition["languages"]
    q = build_query(loc, edition)
    authors = []
    for a in q.get("authors", []):
        if "key" in a:
            authors.append({"key": a["key"]})
        else:
            try:
                ret = ol.new(a, comment="new author")
            except:
                print a
                raise
            print "ret:", ret
            assert isinstance(ret, basestring)
            authors.append({"key": ret})
    q["source_records"] = [loc]
    if authors:
        q["authors"] = authors

    wkey = None
    subjects = subjects_for_work(rec)
    subjects.setdefault("subjects", []).append("Accessible book")
    if "authors" in q:
        wkey = find_matching_work(q)
    if wkey:
        w = ol.get(wkey)
        need_update = False
        for k, subject_list in subjects.items():
            for s in subject_list:
                if s not in w.get(k, []):
                    w.setdefault(k, []).append(s)
                    need_update = True
        if need_update:
            ol.save(wkey, w, "add subjects from new record")
    else:
        w = {"type": "/type/work", "title": q["title"]}
        if "authors" in q:
            w["authors"] = [{"type": "/type/author_role", "author": akey} for akey in q["authors"]]
        w.update(subjects)

        wkey = ol.new(w, comment="initial import")

    q["works"] = [{"key": wkey}]
    for attempt in range(50):
        if attempt > 0:
            print "retrying"
        try:
            pprint(q)
            ret = ol.new(q, comment="initial import")
        except httplib.BadStatusLine:
            sleep(30)
            continue
        except:  # httplib.BadStatusLine
            print q
            raise
        break
    print "ret:", ret
    assert isinstance(ret, basestring)
    key = "/b/" + re_edition_key.match(ret).group(1)
    pool.update(key, q)

    print "add_cover_image"
    t = threading.Thread(target=add_cover_image, args=(ret, ia))
    t.start()
    return

    print "run work finder"

    # too slow
    for a in authors:
        akey = a["key"]
        title_redirects = find_title_redirects(akey)
        works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects)
        works = list(works)
        updated = update_works(akey, works, do_updates=True)
Ejemplo n.º 8
0
def write_edition(loc, edition):
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print(a)
                raise
            print('ret:', ret)
            assert isinstance(ret, basestring)
            #            assert ret['status'] == 'ok'
            #            assert 'created' in ret and len(ret['created']) == 1
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    wkey = None
    subjects = subjects_for_work(rec)
    if 'authors' in q:
        wkey = find_matching_work(q)
    if wkey:
        w = ol.get(wkey)
        need_update = False
        for k, subject_list in subjects.items():
            for s in subject_list:
                if s not in w.get(k, []):
                    w.setdefault(k, []).append(s)
                    need_update = True
        if need_update:
            ol.save(wkey, w, 'add subjects from new record')
    else:
        w = {
            'type': '/type/work',
            'title': q['title'],
        }
        if 'authors' in q:
            w['authors'] = [{
                'type': '/type/author_role',
                'author': akey
            } for akey in q['authors']]
        w.update(subjects)

        wkey = ol.new(w, comment='initial import')
    q['works'] = [{'key': wkey}]

    for attempt in range(5):
        if attempt > 0:
            print('retrying')
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(10)
            continue
        except:  # httplib.BadStatusLine
            print(q)
            raise
        break
    print('ret:', ret)
    assert isinstance(ret, basestring)
    key = '/b/' + re_edition_key.match(ret).group(1)
    #    assert ret['status'] == 'ok'
    #    assert 'created' in ret
    #    editions = [i for i in ret['created'] if i.startswith('/b/OL')]
    #    assert len(editions) == 1
    # get key from return
    pool.update(key, q)

    return

    for a in authors:
        akey = a['key']
        title_redirects = find_title_redirects(akey)
        works = find_works(akey,
                           get_books(akey, books_query(akey)),
                           existing=title_redirects)
        works = list(works)
        updated = update_works(akey, works, do_updates=True)
Ejemplo n.º 9
0
def write_edition(loc, edition):
    q = build_query(loc, edition)
    authors = []
    for a in q.get('authors', []):
        if 'key' in a:
            authors.append({'key': a['key']})
        else:
            try:
                ret = ol.new(a, comment='new author')
            except:
                print(a)
                raise
            print('ret:', ret)
            assert isinstance(ret, six.string_types)
#            assert ret['status'] == 'ok'
#            assert 'created' in ret and len(ret['created']) == 1
            authors.append({'key': ret})
    q['source_records'] = [loc]
    if authors:
        q['authors'] = authors

    wkey = None
    subjects = subjects_for_work(rec)
    if 'authors' in q:
        wkey = find_matching_work(q)
    if wkey:
        w = ol.get(wkey)
        need_update = False
        for k, subject_list in subjects.items():
            for s in subject_list:
                if s not in w.get(k, []):
                    w.setdefault(k, []).append(s)
                    need_update = True
        if need_update:
            ol.save(wkey, w, 'add subjects from new record')
    else:
        w = {
            'type': '/type/work',
            'title': q['title'],
        }
        if 'authors' in q:
            w['authors'] = [{'type':'/type/author_role', 'author': akey} for akey in q['authors']]
        w.update(subjects)

        wkey = ol.new(w, comment='initial import')
    q['works'] = [{'key': wkey}]

    for attempt in range(5):
        if attempt > 0:
            print('retrying')
        try:
            ret = ol.new(q, comment='initial import')
        except httplib.BadStatusLine:
            sleep(10)
            continue
        except: # httplib.BadStatusLine
            print(q)
            raise
        break
    print('ret:', ret)
    assert isinstance(ret, six.string_types)
    key = '/b/' + re_edition_key.match(ret).group(1)
#    assert ret['status'] == 'ok'
#    assert 'created' in ret
#    editions = [i for i in ret['created'] if i.startswith('/b/OL')]
#    assert len(editions) == 1
    # get key from return
    pool.update(key, q)

    return

    for a in authors:
        akey = a['key']
        title_redirects = find_title_redirects(akey)
        works = find_works(akey, get_books(akey, books_query(akey)), existing=title_redirects)
        works = list(works)
        updated = update_works(akey, works, do_updates=True)