def _import_worker(user_key, list_type): """ Called in Task Queue, for importing from douban. Since there may be lots of information to import, it may take some time. Doing this in Task Queue is more proper. @param user_key: DO NOT pass the user, otherwise it would raise exceptions (due to cache??) @param list_type: for subclasses to reuse """ user = auth.user.User.get(user_key) try: all_book_related = douban.get_book_list(user, list_type) except utils.errors.ParseJsonError as err: logging.error("ERROR while importing from Douban, user_key: " + user_key + " list_type: " + list_type) logging.error(err) else: helper = books.SortHelper(user) bl = booklist.BookList.get_or_create(user, list_type) bl.start_importing(len(all_book_related)) # also clear those in memcache helper.clear(list_type) for related in all_book_related: # also added into memcache in merge_into_datastore() b = related.merge_into_datastore(user, update_book=False) if b: # when already such book there, b will be None url, datas = tongji.get_by_isbn(b.isbn) b.set_tongji_info(url, datas) # has to re-get this instance, for it is retrieved inside merge_into_datastore() # the current instance may not be up-to-date bl = booklist.BookList.get_or_create(user, list_type) bl.finish_importing()
def _fetch_parse(self, user, list_type=None): """ It may cause problems to fetch and then parse. Do it together.. @param list_type: which booklist to import. """ if list_type is None: list_type = self.request.get('list_type') if not list_type: return try: datas = douban.get_book_list(user, list_type) except utils.errors.ParseJsonError as err: logging.error("ERROR while importing from Douban, user_key: " + user.key() + " list_type: " + list_type) logging.error(err) self._log(err) return bl = BookList.get_or_create(user, list_type) bl.start_importing(len(datas)) # also clear those in memcache helper = SortHelper(user) helper.clear(list_type) for related in datas: # also added into memcache in merge_into_datastore() b = related.merge_into_datastore(user, update_book=False) if b: # when already such book there, b will be None try: url, datas = tongji.get_by_isbn(b.isbn) b.set_tongji_info(url, datas) except Exception as err: logging.error("ERROR while saving TJ info, isbn: " + b.isbn) logging.error(err) self._log(err) # after all, finish importing bl = BookList.get_or_create(user, list_type) bl.douban_amount = None bl.put() return