def _import_worker(user_key, list_type):
""" Called in Task Queue, for importing from douban.
Since there may be lots of information to import, it may take some time.
Doing this in Task Queue is more proper.
@param user_key: DO NOT pass the user, otherwise it would raise exceptions (due to cache??)
@param list_type: for subclasses to reuse
"""
user = auth.user.User.get(user_key)
try:
all_book_related = douban.get_book_list(user, list_type)
except utils.errors.ParseJsonError as err:
logging.error("ERROR while importing from Douban, user_key: " + user_key +
" list_type: " + list_type)
logging.error(err)
else:
helper = books.SortHelper(user)
bl = booklist.BookList.get_or_create(user, list_type)
bl.start_importing(len(all_book_related))
# also clear those in memcache
helper.clear(list_type)
for related in all_book_related:
# also added into memcache in merge_into_datastore()
b = related.merge_into_datastore(user, update_book=False)
if b:
# when already such book there, b will be None
url, datas = tongji.get_by_isbn(b.isbn)
b.set_tongji_info(url, datas)
# has to re-get this instance, for it is retrieved inside merge_into_datastore()
# the current instance may not be up-to-date
bl = booklist.BookList.get_or_create(user, list_type)
bl.finish_importing()
def _fetch_parse(self, user, list_type=None):
""" It may cause problems to fetch and then parse. Do it together..
@param list_type: which booklist to import.
"""
if list_type is None:
list_type = self.request.get('list_type')
if not list_type:
return
try:
datas = douban.get_book_list(user, list_type)
except utils.errors.ParseJsonError as err:
logging.error("ERROR while importing from Douban, user_key: " + user.key() +
" list_type: " + list_type)
logging.error(err)
self._log(err)
return
bl = BookList.get_or_create(user, list_type)
bl.start_importing(len(datas))
# also clear those in memcache
helper = SortHelper(user)
helper.clear(list_type)
for related in datas:
# also added into memcache in merge_into_datastore()
b = related.merge_into_datastore(user, update_book=False)
if b:
# when already such book there, b will be None
try:
url, datas = tongji.get_by_isbn(b.isbn)
b.set_tongji_info(url, datas)
except Exception as err:
logging.error("ERROR while saving TJ info, isbn: " + b.isbn)
logging.error(err)
self._log(err)
# after all, finish importing
bl = BookList.get_or_create(user, list_type)
bl.douban_amount = None
bl.put()
return
