def load(rec):
"""Given a record, tries to add/match that edition in the system.
Record is a dictionary containing all the metadata of the edition.
The following fields are mandatory:
* title
* source_records
"""
if not rec.get('title'):
raise RequiredField('title')
if not rec.get('source_records'):
raise RequiredField('source_records')
if isinstance(rec['source_records'], basestring):
rec['source_records'] = [rec['source_records']]
edition_pool = build_pool(rec)
if not edition_pool:
return load_data(rec) # 'no books in pool, loading'
#matches = set(item for sublist in edition_pool.values() for item in sublist)
#if len(matches) == 1:
# return {'success': True, 'edition': {'key': list(matches)[0]}}
match = early_exit(rec)
if not match:
match = find_exact_match(rec, edition_pool)
if not match:
rec['full_title'] = rec['title']
if rec.get('subtitle'):
rec['full_title'] += ' ' + rec['subtitle']
e1 = build_marc(rec)
add_db_name(e1)
match = find_match(e1, edition_pool)
if not match: # 'match found:', match, rec['ia']
return load_data(rec)
need_work_save = False
need_edition_save = False
w = None
e = web.ctx.site.get(match)
if e.works:
w = e.works[0].dict()
work_created = False
else:
work_created = True
need_work_save = True
need_edition_save = True
w = {
'type': {
'key': '/type/work'
},
'title': get_title(rec),
'key': web.ctx.site.new_key('/type/work'),
}
e.works = [{'key': w['key']}]
reply = {
'success': True,
'edition': {
'key': match,
'status': 'matched'
},
'work': {
'key': w['key'],
'status': 'matched'
},
}
if not e.get('source_records'):
e['source_records'] = []
existing_source_records = set(e['source_records'])
for i in rec['source_records']:
if i not in existing_source_records:
e['source_records'].append(i)
need_edition_save = True
assert e['source_records']
edits = []
if False and rec.get('authors'):
reply['authors'] = []
east = east_in_by_statement(rec)
work_authors = list(w.get('authors', []))
edition_authors = list(e.authors)
author_in = [import_author(a, eastern=east) for a in rec['authors']]
for a in author_in:
new_author = 'key' not in a
add_to_work = False
add_to_edition = False
if new_author:
a['key'] = web.ctx.site.new_key('/type/author')
assert isinstance(a, dict)
edits.append(a)
add_to_work = True
add_to_edition = True
else:
if not any(i['author'] == a for i in work_authors):
add_to_work = True
if all(i['key'] != a['key'] for i in edition_authors):
add_to_edition = True
if add_to_work:
need_work_save = True
work_authors.append({
'type': {
'key': '/type/author_role'
},
'author': {
'key': a['key']
},
})
if add_to_edition:
need_edition_save = True
edition_authors.append({'key': a['key']})
reply['authors'].append({
'key':
a['key'],
'name':
a['name'],
'status': ('created' if new_author else 'modified'),
})
w['authors'] = work_authors
e['authors'] = edition_authors
if 'subjects' in rec:
work_subjects = list(w.get('subjects', []))
for s in rec['subjects']:
if s not in work_subjects:
work_subjects.append(s)
need_work_save = True
if need_work_save and work_subjects:
w['subjects'] = work_subjects
if 'ocaid' in rec:
new = 'ia:' + rec['ocaid']
if not e.ocaid:
e['ocaid'] = rec['ocaid']
need_edition_save = True
if 'cover' in rec and not e.covers:
cover_url = rec['cover']
cover_id = add_cover(cover_url, e.key)
if cover_id:
e['covers'] = [cover_id]
need_edition_save = True
if not w.get('covers'):
w['covers'] = [cover_id]
need_work_save = True
for f in 'ia_box_id', 'ia_loaded_id':
if f not in rec:
continue
if e.get(f):
assert not isinstance(e[f], basestring)
assert isinstance(e[f], list)
if isinstance(rec[f], basestring):
if rec[f] not in e[f]:
e[f].append(rec[f])
need_edition_save = True
else:
assert isinstance(rec[f], list)
for x in rec[f]:
if x not in e[f]:
e[f].append(x)
need_edition_save = True
if isinstance(rec[f], basestring):
e[f] = [rec[f]]
need_edition_save = True
else:
assert isinstance(rec[f], list)
e[f] = rec[f]
need_edition_save = True
assert not isinstance(e[f], basestring)
assert isinstance(e[f], list)
if need_edition_save:
reply['edition']['status'] = 'modified'
e_dict = e.dict()
assert e_dict and isinstance(e_dict, dict)
edits.append(e_dict)
if need_work_save:
reply['work']['status'] = 'created' if work_created else 'modified'
edits.append(w)
if edits:
for i in edits:
assert i
assert isinstance(i, dict)
web.ctx.site.save_many(edits, 'import new book')
# update_ia_metadata_for_ol_edition(reply['edition']['key'].split('/')[2])
return reply
def load_data(rec):
"""
Adds a new Edition to Open Library. Creates a new Work if required,
otherwise associates the new Edition with an existing Work.
:param dict rec: Edition record to add (no further checks at this point)
:rtype: dict
:return:
{
"success": False,
"error": <error msg>
}
OR
{
"success": True,
"work": {"key": <key>, "status": "created" | "modified" | "matched"},
"edition": {"key": <key>, "status": "created"}
}
"""
cover_url = None
if 'cover' in rec:
cover_url = rec['cover']
del rec['cover']
try:
# get an OL style edition dict
edition = build_query(rec)
except InvalidLanguage as e:
return {
'success': False,
'error': str(e),
}
ekey = web.ctx.site.new_key('/type/edition')
cover_id = None
if cover_url:
cover_id = add_cover(cover_url, ekey)
edition['covers'] = [cover_id]
edits = []
reply = {}
author_in = [
import_author(a, eastern=east_in_by_statement(rec, a))
for a in edition.get('authors', [])
]
# build_author_reply() adds authors to edits
(authors, author_reply) = build_author_reply(author_in, edits)
if authors:
edition['authors'] = authors
reply['authors'] = author_reply
wkey = None
work_state = 'created'
# Look for an existing work
if 'authors' in edition:
wkey = find_matching_work(edition)
if wkey:
w = web.ctx.site.get(wkey)
work_state = 'matched'
found_wkey_match = True
need_update = False
for k in subject_fields:
if k not in rec:
continue
for s in rec[k]:
if s not in w.get(k, []):
w.setdefault(k, []).append(s)
need_update = True
if cover_id:
w.setdefault('covers', []).append(cover_id)
need_update = True
if need_update:
work_state = 'modified'
edits.append(w.dict())
else:
# Create new work
w = new_work(edition, rec, cover_id)
wkey = w['key']
edits.append(w)
assert wkey
edition['works'] = [{'key': wkey}]
edition['key'] = ekey
edits.append(edition)
web.ctx.site.save_many(edits, 'import new book')
# Writes back `openlibrary_edition` and `openlibrary_work` to
# archive.org item after successful import:
update_ia_metadata_for_ol_edition(ekey.split('/')[-1])
reply['success'] = True
reply['edition'] = {'key': ekey, 'status': 'created'}
reply['work'] = {'key': wkey, 'status': work_state}
return reply
def load_data(rec):
cover_url = None
if 'cover' in rec:
cover_url = rec['cover']
del rec['cover']
try:
q = build_query(rec)
except InvalidLanguage as e:
return {
'success': False,
'error': str(e),
}
edits = []
reply = {}
author_in = [
import_author(a, eastern=east_in_by_statement(rec, a))
for a in q.get('authors', [])
]
(authors, author_reply) = build_author_reply(author_in, edits)
#q['source_records'] = [loc]
if authors:
q['authors'] = authors
reply['authors'] = author_reply
wkey = None
ekey = web.ctx.site.new_key('/type/edition')
cover_id = None
if cover_url:
cover_id = add_cover(cover_url, ekey)
q['covers'] = [cover_id]
work_state = 'created'
if 'authors' in q:
wkey = find_matching_work(q)
if wkey:
w = web.ctx.site.get(wkey)
work_state = 'matched'
found_wkey_match = True
need_update = False
for k in subject_fields:
if k not in rec:
continue
for s in rec[k]:
if s not in w.get(k, []):
w.setdefault(k, []).append(s)
need_update = True
if cover_id:
w.setdefault('covers', []).append(cover_id)
need_update = True
if need_update:
work_state = 'modified'
w_dict = w.dict()
assert w_dict and isinstance(w_dict, dict)
edits.append(w_dict)
else:
w = new_work(q, rec, cover_id)
wkey = w['key']
edits.append(w)
assert wkey
q['works'] = [{'key': wkey}]
q['key'] = ekey
assert isinstance(q, dict)
edits.append(q)
assert edits
web.ctx.site.save_many(edits, 'import new book')
reply['success'] = True
reply['edition'] = {
'key': ekey,
'status': 'created',
}
reply['work'] = {
'key': wkey,
'status': work_state,
}
return reply
def load(rec):
"""Given a record, tries to add/match that edition in the system.
Record is a dictionary containing all the metadata of the edition.
The following fields are mandatory:
* title
* source_records
"""
if not rec.get('title'):
raise RequiredField('title')
if not rec.get('source_records'):
raise RequiredField('source_records')
if isinstance(rec['source_records'], basestring):
rec['source_records'] = [rec['source_records']]
edition_pool = build_pool(rec)
if not edition_pool:
return load_data(rec) # 'no books in pool, loading'
#matches = set(item for sublist in edition_pool.values() for item in sublist)
#if len(matches) == 1:
# return {'success': True, 'edition': {'key': list(matches)[0]}}
match = early_exit(rec)
if not match:
match = find_exact_match(rec, edition_pool)
if not match:
rec['full_title'] = rec['title']
if rec.get('subtitle'):
rec['full_title'] += ' ' + rec['subtitle']
e1 = build_marc(rec)
add_db_name(e1)
match = find_match(e1, edition_pool)
if not match: # 'match found:', match, rec['ia']
return load_data(rec)
need_work_save = False
need_edition_save = False
w = None
e = web.ctx.site.get(match)
if e.works:
w = e.works[0].dict()
work_created = False
else:
work_created = True
need_work_save = True
need_edition_save = True
w = {
'type': {'key': '/type/work'},
'title': get_title(rec),
'key': web.ctx.site.new_key('/type/work'),
}
e.works = [{'key': w['key']}]
reply = {
'success': True,
'edition': {'key': match, 'status': 'matched'},
'work': {'key': w['key'], 'status': 'matched'},
}
if not e.get('source_records'):
e['source_records'] = []
existing_source_records = set(e['source_records'])
for i in rec['source_records']:
if i not in existing_source_records:
e['source_records'].append(i)
need_edition_save = True
assert e['source_records']
edits = []
if False and rec.get('authors'):
reply['authors'] = []
east = east_in_by_statement(rec)
work_authors = list(w.get('authors', []))
edition_authors = list(e.authors)
author_in = [import_author(a, eastern=east) for a in rec['authors']]
for a in author_in:
new_author = 'key' not in a
add_to_work = False
add_to_edition = False
if new_author:
a['key'] = web.ctx.site.new_key('/type/author')
assert isinstance(a, dict)
edits.append(a)
add_to_work = True
add_to_edition = True
else:
if not any(i['author'] == a for i in work_authors):
add_to_work = True
if all(i['key'] != a['key'] for i in edition_authors):
add_to_edition = True
if add_to_work:
need_work_save = True
work_authors.append({
'type': {'key': '/type/author_role'},
'author': {'key': a['key'] },
})
if add_to_edition:
need_edition_save = True
edition_authors.append({'key': a['key'] })
reply['authors'].append({
'key': a['key'],
'name': a['name'],
'status': ('created' if new_author else 'modified'),
})
w['authors'] = work_authors
e['authors'] = edition_authors
if 'subjects' in rec:
work_subjects = list(w.get('subjects', []))
for s in rec['subjects']:
if s not in work_subjects:
work_subjects.append(s)
need_work_save = True
if need_work_save and work_subjects:
w['subjects'] = work_subjects
if 'ocaid' in rec:
new = 'ia:' + rec['ocaid']
if not e.ocaid:
e['ocaid'] = rec['ocaid']
need_edition_save = True
if 'cover' in rec and not e.covers:
cover_url = rec['cover']
cover_id = add_cover(cover_url, e.key)
if cover_id:
e['covers'] = [cover_id]
need_edition_save = True
if not w.get('covers'):
w['covers'] = [cover_id]
need_work_save = True
for f in 'ia_box_id', 'ia_loaded_id':
if f not in rec:
continue
if e.get(f):
assert not isinstance(e[f], basestring)
assert isinstance(e[f], list)
if isinstance(rec[f], basestring):
if rec[f] not in e[f]:
e[f].append(rec[f])
need_edition_save = True
else:
assert isinstance(rec[f], list)
for x in rec[f]:
if x not in e[f]:
e[f].append(x)
need_edition_save = True
if isinstance(rec[f], basestring):
e[f] = [rec[f]]
need_edition_save = True
else:
assert isinstance(rec[f], list)
e[f] = rec[f]
need_edition_save = True
assert not isinstance(e[f], basestring)
assert isinstance(e[f], list)
if need_edition_save:
reply['edition']['status'] = 'modified'
e_dict = e.dict()
assert e_dict and isinstance(e_dict, dict)
edits.append(e_dict)
if need_work_save:
reply['work']['status'] = 'created' if work_created else 'modified'
edits.append(w)
if edits:
for i in edits:
assert i
assert isinstance(i, dict)
web.ctx.site.save_many(edits, 'import new book')
return reply
def load_data(rec):
cover_url = None
if 'cover' in rec:
cover_url = rec['cover']
del rec['cover']
try:
q = build_query(rec)
except InvalidLanguage as e:
return {
'success': False,
'error': str(e),
}
edits = []
reply = {}
author_in = [import_author(a, eastern=east_in_by_statement(rec, a)) for a in q.get('authors', [])]
(authors, author_reply) = build_author_reply(author_in, edits)
#q['source_records'] = [loc]
if authors:
q['authors'] = authors
reply['authors'] = author_reply
wkey = None
ekey = web.ctx.site.new_key('/type/edition')
cover_id = None
if cover_url:
cover_id = add_cover(cover_url, ekey)
q['covers'] = [cover_id]
work_state = 'created'
if 'authors' in q:
wkey = find_matching_work(q)
if wkey:
w = web.ctx.site.get(wkey)
work_state = 'matched'
found_wkey_match = True
need_update = False
for k in subject_fields:
if k not in rec:
continue
for s in rec[k]:
if s not in w.get(k, []):
w.setdefault(k, []).append(s)
need_update = True
if cover_id:
w.setdefault('covers', []).append(cover_id)
need_update = True
if need_update:
work_state = 'modified'
w_dict = w.dict()
assert w_dict and isinstance(w_dict, dict)
edits.append(w_dict)
else:
w = new_work(q, rec, cover_id)
wkey = w['key']
edits.append(w)
assert wkey
q['works'] = [{'key': wkey}]
q['key'] = ekey
assert isinstance(q, dict)
edits.append(q)
assert edits
web.ctx.site.save_many(edits, 'import new book')
reply['success'] = True
reply['edition'] = { 'key': ekey, 'status': 'created', }
reply['work'] = { 'key': wkey, 'status': work_state, }
return reply
def load_data(rec):
"""
Adds a new Edition to Open Library. Creates a new Work if required,
otherwise associates the new Edition with an existing Work.
:param dict rec: Edition record to add (no further checks at this point)
:rtype: dict
:return:
{
"success": False,
"error": <error msg>
}
OR
{
"success": True,
"work": {"key": <key>, "status": "created" | "modified" | "matched"},
"edition": {"key": <key>, "status": "created"}
}
"""
cover_url = None
if 'cover' in rec:
cover_url = rec['cover']
del rec['cover']
try:
# get an OL style edition dict
edition = build_query(rec)
except InvalidLanguage as e:
return {
'success': False,
'error': str(e),
}
ekey = web.ctx.site.new_key('/type/edition')
cover_id = None
if cover_url:
cover_id = add_cover(cover_url, ekey)
edition['covers'] = [cover_id]
edits = []
reply = {}
author_in = [import_author(a, eastern=east_in_by_statement(rec, a)) for a in edition.get('authors', [])]
# build_author_reply() adds authors to edits
(authors, author_reply) = build_author_reply(author_in, edits)
if authors:
edition['authors'] = authors
reply['authors'] = author_reply
wkey = None
work_state = 'created'
# Look for an existing work
if 'authors' in edition:
wkey = find_matching_work(edition)
if wkey:
w = web.ctx.site.get(wkey)
work_state = 'matched'
found_wkey_match = True
need_update = False
for k in subject_fields:
if k not in rec:
continue
for s in rec[k]:
if s not in w.get(k, []):
w.setdefault(k, []).append(s)
need_update = True
if cover_id:
w.setdefault('covers', []).append(cover_id)
need_update = True
if need_update:
work_state = 'modified'
edits.append(w.dict())
else:
# Create new work
w = new_work(edition, rec, cover_id)
wkey = w['key']
edits.append(w)
assert wkey
edition['works'] = [{'key': wkey}]
edition['key'] = ekey
edits.append(edition)
web.ctx.site.save_many(edits, 'import new book')
# Writes back `openlibrary_edition` and `openlibrary_work` to
# archive.org item after successful import:
update_ia_metadata_for_ol_edition(ekey.split('/')[-1])
reply['success'] = True
reply['edition'] = {'key': ekey, 'status': 'created'}
reply['work'] = {'key': wkey, 'status': work_state}
return reply
def load(rec):
if not rec.get('title'):
raise RequiredField('title')
edition_pool = build_pool(rec)
#print 'pool:', edition_pool
if not edition_pool:
return load_data(rec) # 'no books in pool, loading'
#matches = set(item for sublist in edition_pool.values() for item in sublist)
#if len(matches) == 1:
# return {'success': True, 'edition': {'key': list(matches)[0]}}
match = find_exact_match(rec, edition_pool)
if not match:
rec['full_title'] = rec['title']
if rec.get('subtitle'):
rec['full_title'] += ' ' + rec['subtitle']
e1 = build_marc(rec)
add_db_name(e1)
#print
#print 'e1', e1
#print
#print 'pool', edition_pool
match = find_match(e1, edition_pool)
if match: # 'match found:', match, rec['ia']
e = web.ctx.site.get(match)
w = e['works'][0]
reply = {
'success': True,
'edition': {
'key': match,
'status': 'matched'
},
'work': {
'key': w.key,
'status': 'matched'
},
}
edits = []
need_work_save = False
need_edition_save = False
if rec.get('authors'):
reply['authors'] = []
east = east_in_by_statement(rec)
work_authors = list(w.authors)
edition_authors = list(e.authors)
author_in = [
import_author(a, eastern=east) for a in rec['authors']
]
for a in author_in:
new_author = 'key' not in a
add_to_work = False
add_to_edition = False
if new_author:
a['key'] = web.ctx.site.new_key('/type/author')
edits.append(a)
add_to_work = True
add_to_edition = True
else:
if not any(i.author.key == a['key'] for i in work_authors):
add_to_work = True
if not any(i.key == a['key'] for i in edition_authors):
add_to_edition = True
if add_to_work:
need_work_save = True
work_authors.append({
'type': {
'key': '/type/author_role'
},
'author': {
'key': a['key']
},
})
if add_to_edition:
need_edition_save = True
edition_authors.append({'key': a['key']})
reply['authors'].append({
'key':
a['key'],
'name':
a['name'],
'status': ('created' if new_author else 'modified'),
})
w.authors = work_authors
e.authors = edition_authors
if 'subjects' in rec:
work_subjects = list(w.subjects)
for s in rec['subjects']:
if s not in w.subjects:
#print 'w.subjects.append(%s)' % s
work_subjects.append(s)
need_work_save = True
if need_work_save:
w.subjects = work_subjects
if need_edition_save:
reply['edition']['status'] = 'modified'
edits.append(e)
web.ctx.site.save(e, match, 'update edition')
if need_work_save:
reply['work']['status'] = 'modified'
edits.append(w)
if edits:
web.ctx.site.save_many(edits, 'import new book')
return reply
#add_source_records(match, ia)
else: # 'no match found', rec['ia']
return load_data(rec)
def load(rec):
if not rec.get('title'):
raise RequiredField('title')
edition_pool = build_pool(rec)
#print 'pool:', edition_pool
if not edition_pool:
return load_data(rec) # 'no books in pool, loading'
#matches = set(item for sublist in edition_pool.values() for item in sublist)
#if len(matches) == 1:
# return {'success': True, 'edition': {'key': list(matches)[0]}}
match = find_exact_match(rec, edition_pool)
if not match:
rec['full_title'] = rec['title']
if rec.get('subtitle'):
rec['full_title'] += ' ' + rec['subtitle']
e1 = build_marc(rec)
add_db_name(e1)
#print
#print 'e1', e1
#print
#print 'pool', edition_pool
match = find_match(e1, edition_pool)
if match: # 'match found:', match, rec['ia']
e = web.ctx.site.get(match)
w = e['works'][0]
reply = {
'success': True,
'edition': {'key': match, 'status': 'matched'},
'work': {'key': w.key, 'status': 'matched'},
}
need_work_save = False
need_edition_save = False
if rec.get('authors'):
reply['authors'] = []
east = east_in_by_statement(rec)
work_authors = list(w.authors)
edition_authors = list(e.authors)
author_in = [import_author(a, eastern=east) for a in rec['authors']]
for a in author_in:
new_author = 'key' not in a
add_to_work = False
add_to_edition = False
if new_author:
a['key'] = web.ctx.site.new_key('/type/author')
aobj = web.ctx.site.save(a, comment='new author')
add_to_work = True
add_to_edition = True
else:
if not any(i.author.key == a['key'] for i in work_authors):
add_to_work = True
if not any(i.key == a['key'] for i in edition_authors):
add_to_edition = True
if add_to_work:
need_work_save = True
work_authors.append({
'type': {'key': '/type/author_role'},
'author': {'key': a['key'] },
})
if add_to_edition:
need_edition_save = True
edition_authors.append({'key': a['key'] })
reply['authors'].append({
'key': a['key'],
'name': a['name'],
'status': ('created' if new_author else 'modified'),
})
w.authors = work_authors
e.authors = edition_authors
if 'subjects' in rec:
work_subjects = list(w.subjects)
for s in rec['subjects']:
if s not in w.subjects:
#print 'w.subjects.append(%s)' % s
work_subjects.append(s)
need_work_save = True
if need_work_save:
w.subjects = work_subjects
if need_edition_save:
reply['edition']['status'] = 'modified'
web.ctx.site.save(e, match, 'update edition')
if need_work_save:
reply['work']['status'] = 'modified'
web.ctx.site.save(w, w.key, 'update work')
return reply
#add_source_records(match, ia)
else: # 'no match found', rec['ia']
return load_data(rec)
